xref: /titanic_52/usr/src/uts/sun4/os/startup.c (revision 80feccabd0bc0fca1df2b60bfa50449d9a6e3e09)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/machsystm.h>
30 #include <sys/archsystm.h>
31 #include <sys/vm.h>
32 #include <sys/cpu.h>
33 #include <sys/atomic.h>
34 #include <sys/reboot.h>
35 #include <sys/kdi.h>
36 #include <sys/bootconf.h>
37 #include <sys/memlist_plat.h>
38 #include <sys/memlist_impl.h>
39 #include <sys/prom_plat.h>
40 #include <sys/prom_isa.h>
41 #include <sys/autoconf.h>
42 #include <sys/intreg.h>
43 #include <sys/ivintr.h>
44 #include <sys/fpu/fpusystm.h>
45 #include <sys/iommutsb.h>
46 #include <vm/vm_dep.h>
47 #include <vm/seg_dev.h>
48 #include <vm/seg_kmem.h>
49 #include <vm/seg_kpm.h>
50 #include <vm/seg_map.h>
51 #include <vm/seg_kp.h>
52 #include <sys/sysconf.h>
53 #include <vm/hat_sfmmu.h>
54 #include <sys/kobj.h>
55 #include <sys/sun4asi.h>
56 #include <sys/clconf.h>
57 #include <sys/platform_module.h>
58 #include <sys/panic.h>
59 #include <sys/cpu_sgnblk_defs.h>
60 #include <sys/clock.h>
61 #include <sys/cmn_err.h>
62 #include <sys/promif.h>
63 #include <sys/prom_debug.h>
64 #include <sys/traptrace.h>
65 #include <sys/memnode.h>
66 #include <sys/mem_cage.h>
67 #include <sys/mmu.h>
68 
69 extern void setup_trap_table(void);
70 extern int cpu_intrq_setup(struct cpu *);
71 extern void cpu_intrq_register(struct cpu *);
72 extern void contig_mem_init(void);
73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
74 extern void mach_dump_buffer_init(void);
75 extern void mach_descrip_init(void);
76 extern void mach_descrip_startup_fini(void);
77 extern void mach_memscrub(void);
78 extern void mach_fpras(void);
79 extern void mach_cpu_halt_idle(void);
80 extern void mach_hw_copy_limit(void);
81 extern void load_mach_drivers(void);
82 extern void load_tod_module(void);
83 #pragma weak load_tod_module
84 
85 extern int ndata_alloc_mmfsa(struct memlist *ndata);
86 #pragma weak ndata_alloc_mmfsa
87 
88 extern void cif_init(void);
89 #pragma weak cif_init
90 
91 extern void parse_idprom(void);
92 extern void add_vx_handler(char *, int, void (*)(cell_t *));
93 extern void mem_config_init(void);
94 extern void memseg_remap_init(void);
95 
96 extern void mach_kpm_init(void);
97 extern int size_pse_array(pgcnt_t, int);
98 
99 /*
100  * External Data:
101  */
102 extern int vac_size;	/* cache size in bytes */
103 extern uint_t vac_mask;	/* VAC alignment consistency mask */
104 extern uint_t vac_colors;
105 
106 /*
107  * Global Data Definitions:
108  */
109 
110 /*
111  * XXX - Don't port this to new architectures
112  * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
113  * 'romp' has no use with a prom with an IEEE 1275 client interface.
114  * The driver doesn't use the value, but it depends on the symbol.
115  */
116 void *romp;		/* veritas driver won't load without romp 4154976 */
117 /*
118  * Declare these as initialized data so we can patch them.
119  */
120 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
121 pgcnt_t segkpsize =
122     btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
123 uint_t segmap_percent = 12; /* Size of segmap segment */
124 
125 int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
126 int vac_copyback = 1;
127 char *cache_mode = NULL;
128 int use_mix = 1;
129 int prom_debug = 0;
130 
131 caddr_t boot_tba;		/* %tba at boot - used by kmdb */
132 uint_t	tba_taken_over = 0;
133 
134 caddr_t s_text;			/* start of kernel text segment */
135 caddr_t e_text;			/* end of kernel text segment */
136 caddr_t s_data;			/* start of kernel data segment */
137 caddr_t e_data;			/* end of kernel data segment */
138 
139 caddr_t modtext;		/* beginning of module text */
140 size_t	modtext_sz;		/* size of module text */
141 caddr_t moddata;		/* beginning of module data reserve */
142 caddr_t e_moddata;		/* end of module data reserve */
143 
144 /*
145  * End of first block of contiguous kernel in 32-bit virtual address space
146  */
147 caddr_t		econtig32;	/* end of first blk of contiguous kernel */
148 
149 caddr_t		ncbase;		/* beginning of non-cached segment */
150 caddr_t		ncend;		/* end of non-cached segment */
151 
152 size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
153 caddr_t	nalloc_base;		/* beginning of nucleus allocation */
154 caddr_t nalloc_end;		/* end of nucleus allocatable memory */
155 caddr_t valloc_base;		/* beginning of kvalloc segment	*/
156 
157 caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
158 caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
159 size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
160 caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
161 int	kmem64_szc;		/* page size code */
162 uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
163 
164 uintptr_t shm_alignment;	/* VAC address consistency modulus */
165 struct memlist *phys_install;	/* Total installed physical memory */
166 struct memlist *phys_avail;	/* Available (unreserved) physical memory */
167 struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
168 struct memlist *nopp_list;	/* pages with no backing page structs */
169 struct memlist ndata;		/* memlist of nucleus allocatable memory */
170 int memexp_flag;		/* memory expansion card flag */
171 uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
172 pgcnt_t obp_pages;		/* Physical pages used by OBP */
173 
174 /*
175  * VM data structures
176  */
177 long page_hashsz;		/* Size of page hash table (power of two) */
178 struct page *pp_base;		/* Base of system page struct array */
179 size_t pp_sz;			/* Size in bytes of page struct array */
180 struct page **page_hash;	/* Page hash table */
181 pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
182 size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
183 int pse_shift;			/* log2(pse_table_size) */
184 struct seg ktextseg;		/* Segment used for kernel executable image */
185 struct seg kvalloc;		/* Segment used for "valloc" mapping */
186 struct seg kpseg;		/* Segment used for pageable kernel virt mem */
187 struct seg ktexthole;		/* Segment used for nucleus text hole */
188 struct seg kmapseg;		/* Segment used for generic kernel mappings */
189 struct seg kpmseg;		/* Segment used for physical mapping */
190 struct seg kdebugseg;		/* Segment used for the kernel debugger */
191 
192 void *kpm_pp_base;		/* Base of system kpm_page array */
193 size_t	kpm_pp_sz;		/* Size of system kpm_page array */
194 pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
195 
196 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
197 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
198 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
199 
200 int segzio_fromheap = 0;	/* zio allocations occur from heap */
201 caddr_t segzio_base;		/* Base address of segzio */
202 pgcnt_t segziosize = 0;		/* size of zio segment in pages */
203 
204 /*
205  * debugger pages (if allocated)
206  */
207 struct vnode kdebugvp;
208 
209 /*
210  * VA range available to the debugger
211  */
212 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
213 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
214 
215 /*
216  * Segment for relocated kernel structures in 64-bit large RAM kernels
217  */
218 struct seg kmem64;
219 
220 struct memseg *memseg_free;
221 
222 struct vnode unused_pages_vp;
223 
224 /*
225  * VM data structures allocated early during boot.
226  */
227 size_t pagehash_sz;
228 uint64_t memlist_sz;
229 
230 char tbr_wr_addr_inited = 0;
231 
232 caddr_t	mpo_heap32_buf = NULL;
233 size_t	mpo_heap32_bufsz = 0;
234 
235 /*
236  * Static Routines:
237  */
238 static int ndata_alloc_memseg(struct memlist *, size_t);
239 static void memlist_new(uint64_t, uint64_t, struct memlist **);
240 static void memlist_add(uint64_t, uint64_t,
241 	struct memlist **, struct memlist **);
242 static void kphysm_init(void);
243 static void kvm_init(void);
244 static void install_kmem64_tte(void);
245 
246 static void startup_init(void);
247 static void startup_memlist(void);
248 static void startup_modules(void);
249 static void startup_bop_gone(void);
250 static void startup_vm(void);
251 static void startup_end(void);
252 static void setup_cage_params(void);
253 static void startup_create_io_node(void);
254 
255 static pgcnt_t npages;
256 static struct memlist *memlist;
257 void *memlist_end;
258 
259 static pgcnt_t bop_alloc_pages;
260 static caddr_t hblk_base;
261 uint_t hblk_alloc_dynamic = 0;
262 uint_t hblk1_min = H1MIN;
263 
264 
265 /*
266  * Hooks for unsupported platforms and down-rev firmware
267  */
268 int iam_positron(void);
269 #pragma weak iam_positron
270 static void do_prom_version_check(void);
271 
272 /*
273  * After receiving a thermal interrupt, this is the number of seconds
274  * to delay before shutting off the system, assuming
275  * shutdown fails.  Use /etc/system to change the delay if this isn't
276  * large enough.
277  */
278 int thermal_powerdown_delay = 1200;
279 
280 /*
281  * Used to hold off page relocations into the cage until OBP has completed
282  * its boot-time handoff of its resources to the kernel.
283  */
284 int page_relocate_ready = 0;
285 
286 /*
287  * Enable some debugging messages concerning memory usage...
288  */
289 #ifdef  DEBUGGING_MEM
290 static int debugging_mem;
291 static void
292 printmemlist(char *title, struct memlist *list)
293 {
294 	if (!debugging_mem)
295 		return;
296 
297 	printf("%s\n", title);
298 
299 	while (list) {
300 		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
301 		    (uint32_t)(list->address >> 32), (uint32_t)list->address,
302 		    (uint32_t)(list->size >> 32), (uint32_t)(list->size));
303 		list = list->next;
304 	}
305 }
306 
307 void
308 printmemseg(struct memseg *memseg)
309 {
310 	if (!debugging_mem)
311 		return;
312 
313 	printf("memseg\n");
314 
315 	while (memseg) {
316 		prom_printf("\tpage = 0x%p, epage = 0x%p, "
317 		    "pfn = 0x%x, epfn = 0x%x\n",
318 		    memseg->pages, memseg->epages,
319 		    memseg->pages_base, memseg->pages_end);
320 		memseg = memseg->next;
321 	}
322 }
323 
324 #define	debug_pause(str)	halt((str))
325 #define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
326 #define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
327 #define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
328 #define	MPRINTF3(str, a, b, c) \
329 	if (debugging_mem) prom_printf((str), (a), (b), (c))
330 #else	/* DEBUGGING_MEM */
331 #define	MPRINTF(str)
332 #define	MPRINTF1(str, a)
333 #define	MPRINTF2(str, a, b)
334 #define	MPRINTF3(str, a, b, c)
335 #endif	/* DEBUGGING_MEM */
336 
337 
338 /*
339  *
340  *                    Kernel's Virtual Memory Layout.
341  *                       /-----------------------\
342  * 0xFFFFFFFF.FFFFFFFF  -|                       |-
343  *                       |   OBP's virtual page  |
344  *                       |        tables         |
345  * 0xFFFFFFFC.00000000  -|-----------------------|-
346  *                       :                       :
347  *                       :                       :
348  *                      -|-----------------------|-
349  *                       |       segzio          | (base and size vary)
350  * 0xFFFFFE00.00000000  -|-----------------------|-
351  *                       |                       |  Ultrasparc I/II support
352  *                       |    segkpm segment     |  up to 2TB of physical
353  *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
354  *                       |                       |
355  * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
356  *                       :                       :
357  *                       :                       :
358  * 0xFFFFF810.00000000  -|-----------------------|- hole_end
359  *                       |                       |      ^
360  *                       |  UltraSPARC I/II call |      |
361  *                       | bug requires an extra |      |
362  *                       | 4 GB of space between |      |
363  *                       |   hole and used RAM   |	|
364  *                       |                       |      |
365  * 0xFFFFF800.00000000  -|-----------------------|-     |
366  *                       |                       |      |
367  *                       | Virtual Address Hole  |   UltraSPARC
368  *                       |  on UltraSPARC I/II   |  I/II * ONLY *
369  *                       |                       |      |
370  * 0x00000800.00000000  -|-----------------------|-     |
371  *                       |                       |      |
372  *                       |  UltraSPARC I/II call |      |
373  *                       | bug requires an extra |      |
374  *                       | 4 GB of space between |      |
375  *                       |   hole and used RAM   |      |
376  *                       |                       |      v
377  * 0x000007FF.00000000  -|-----------------------|- hole_start -----
378  *                       :                       :		   ^
379  *                       :                       :		   |
380  *                       |-----------------------|                 |
381  *                       |                       |                 |
382  *                       |  ecache flush area    |                 |
383  *                       |  (twice largest e$)   |                 |
384  *                       |                       |                 |
385  * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
386  *                       | overmapped area       |   alignend_end  |
387  *                       | (kmem64_alignsize     |		   |
388  *                       |  boundary)            |		   |
389  * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
390  *                       |                       |		   |
391  *                       |   64-bit kernel ONLY  |		   |
392  *                       |                       |		   |
393  *                       |    kmem64 segment     |		   |
394  *                       |                       |		   |
395  *                       | (Relocated extra HME  |	     Approximately
396  *                       |   block allocations,  |	    1 TB of virtual
397  *                       |   memnode freelists,  |	     address space
398  *                       |    HME hash buckets,  |		   |
399  *                       | mml_table, kpmp_table,|		   |
400  *                       |  page_t array and     |		   |
401  *                       |  hashblock pool to    |		   |
402  *                       |   avoid hard-coded    |		   |
403  *                       |     32-bit vaddr      |		   |
404  *                       |     limitations)      |		   |
405  *                       |                       |		   v
406  * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
407  *                       |                       |
408  *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
409  *                       |                       |
410  * 0x00000300.00000000  -|-----------------------|- SYSBASE
411  *                       :                       :
412  *                       :                       :
413  *                      -|-----------------------|-
414  *                       |                       |
415  *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
416  *                       |                       |               256G MAX)
417  * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
418  *                       :                       :
419  *                       :                       :
420  *                      -|-----------------------|-
421  *                       |                       |
422  *                       |       segkp           |    SEGKPSIZE (2GB)
423  *                       |                       |
424  *                       |                       |
425  * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
426  *                       |                       |
427  * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
428  *                       |                       |       (SEGKPBASE - 0x400000)
429  * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
430  *                       |                       |       (MEMSCRUBBASE - NCARGS)
431  * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
432  *                       |                       |       (ARGSBASE - PPMAPSIZE)
433  * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
434  *                       |                       |
435  * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
436  *                       |                       |
437  * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
438  *                       :                       :
439  *                       :                       :
440  * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
441  *                       |                       |
442  *                       |         OBP           |
443  *                       |                       |
444  * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
445  *                       |         kmdb          |
446  * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
447  *                       :                       :
448  *                       :                       :
449  * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
450  *                       |                       |
451  *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
452  *                       |                       |    ~64MB)
453  * 0x00000000.70002000  -|-----------------------|
454  *                       |     panicbuf          |
455  * 0x00000000.70000000  -|-----------------------|- SYSBASE32
456  *                       |       boot-time       |
457  *                       |    temporary space    |
458  * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
459  *                       :                       :
460  *                       :                       :
461  *                       |                       |
462  *                       |-----------------------|- econtig32
463  *                       |    vm structures      |
464  * 0x00000000.01C00000   |-----------------------|- nalloc_end
465  *                       |         TSBs          |
466  *                       |-----------------------|- end/nalloc_base
467  *                       |   kernel data & bss   |
468  * 0x00000000.01800000  -|-----------------------|
469  *                       :   nucleus text hole   :
470  * 0x00000000.01400000  -|-----------------------|
471  *                       :                       :
472  *                       |-----------------------|
473  *                       |      module text      |
474  *                       |-----------------------|- e_text/modtext
475  *                       |      kernel text      |
476  *                       |-----------------------|
477  *                       |    trap table (48k)   |
478  * 0x00000000.01000000  -|-----------------------|- KERNELBASE
479  *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
480  *                       |-----------------------|
481  *                       |                       |
482  *                       |        invalid        |
483  *                       |                       |
484  * 0x00000000.00000000  _|_______________________|
485  *
486  *
487  *
488  *                   32-bit User Virtual Memory Layout.
489  *                       /-----------------------\
490  *                       |                       |
491  *                       |        invalid        |
492  *                       |                       |
493  *          0xFFC00000  -|-----------------------|- USERLIMIT
494  *                       |       user stack      |
495  *                       :                       :
496  *                       :                       :
497  *                       :                       :
498  *                       |       user data       |
499  *                      -|-----------------------|-
500  *                       |       user text       |
501  *          0x00002000  -|-----------------------|-
502  *                       |       invalid         |
503  *          0x00000000  _|_______________________|
504  *
505  *
506  *
507  *                   64-bit User Virtual Memory Layout.
508  *                       /-----------------------\
509  *                       |                       |
510  *                       |        invalid        |
511  *                       |                       |
512  *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
513  *                       |       user stack      |
514  *                       :                       :
515  *                       :                       :
516  *                       :                       :
517  *                       |       user data       |
518  *                      -|-----------------------|-
519  *                       |       user text       |
520  *  0x00000000.01000000 -|-----------------------|-
521  *                       |       invalid         |
522  *  0x00000000.00000000 _|_______________________|
523  */
524 
525 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
526 extern uint64_t ecache_flush_address(void);
527 
528 #pragma weak load_platform_modules
529 #pragma weak plat_startup_memlist
530 #pragma weak ecache_init_scrub_flush_area
531 #pragma weak ecache_flush_address
532 
533 
534 /*
535  * By default the DR Cage is enabled for maximum OS
536  * MPSS performance.  Users needing to disable the cage mechanism
537  * can set this variable to zero via /etc/system.
538  * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
539  * will result in loss of DR functionality.
540  * Platforms wishing to disable kernel Cage by default
541  * should do so in their set_platform_defaults() routine.
542  */
543 int	kernel_cage_enable = 1;
544 
545 static void
546 setup_cage_params(void)
547 {
548 	void (*func)(void);
549 
550 	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
551 	if (func != NULL) {
552 		(*func)();
553 		return;
554 	}
555 
556 	if (kernel_cage_enable == 0) {
557 		return;
558 	}
559 	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
560 
561 	if (kcage_on) {
562 		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
563 	} else {
564 		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
565 	}
566 
567 }
568 
569 /*
570  * Machine-dependent startup code
571  */
572 void
573 startup(void)
574 {
575 	startup_init();
576 	if (&startup_platform)
577 		startup_platform();
578 	startup_memlist();
579 	startup_modules();
580 	setup_cage_params();
581 	startup_bop_gone();
582 	startup_vm();
583 	startup_end();
584 }
585 
586 struct regs sync_reg_buf;
587 uint64_t sync_tt;
588 
589 void
590 sync_handler(void)
591 {
592 	struct  panic_trap_info 	ti;
593 	int i;
594 
595 	/*
596 	 * Prevent trying to talk to the other CPUs since they are
597 	 * sitting in the prom and won't reply.
598 	 */
599 	for (i = 0; i < NCPU; i++) {
600 		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
601 			cpu[i]->cpu_flags &= ~CPU_READY;
602 			cpu[i]->cpu_flags |= CPU_QUIESCED;
603 			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
604 		}
605 	}
606 
607 	/*
608 	 * We've managed to get here without going through the
609 	 * normal panic code path. Try and save some useful
610 	 * information.
611 	 */
612 	if (!panicstr && (curthread->t_panic_trap == NULL)) {
613 		ti.trap_type = sync_tt;
614 		ti.trap_regs = &sync_reg_buf;
615 		ti.trap_addr = NULL;
616 		ti.trap_mmu_fsr = 0x0;
617 
618 		curthread->t_panic_trap = &ti;
619 	}
620 
621 	/*
622 	 * If we're re-entering the panic path, update the signature
623 	 * block so that the SC knows we're in the second part of panic.
624 	 */
625 	if (panicstr)
626 		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
627 
628 	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
629 	panic("sync initiated");
630 }
631 
632 
633 static void
634 startup_init(void)
635 {
636 	/*
637 	 * We want to save the registers while we're still in OBP
638 	 * so that we know they haven't been fiddled with since.
639 	 * (In principle, OBP can't change them just because it
640 	 * makes a callback, but we'd rather not depend on that
641 	 * behavior.)
642 	 */
643 	char		sync_str[] =
644 	    "warning @ warning off : sync "
645 	    "%%tl-c %%tstate h# %p x! "
646 	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
647 	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
648 	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
649 	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
650 	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
651 	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
652 	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
653 	    "sync ; warning !";
654 
655 	/*
656 	 * 20 == num of %p substrings
657 	 * 16 == max num of chars %p will expand to.
658 	 */
659 	char 		bp[sizeof (sync_str) + 16 * 20];
660 
661 	/*
662 	 * Initialize ptl1 stack for the 1st CPU.
663 	 */
664 	ptl1_init_cpu(&cpu0);
665 
666 	/*
667 	 * Initialize the address map for cache consistent mappings
668 	 * to random pages; must be done after vac_size is set.
669 	 */
670 	ppmapinit();
671 
672 	/*
673 	 * Initialize the PROM callback handler.
674 	 */
675 	init_vx_handler();
676 
677 	/*
678 	 * have prom call sync_callback() to handle the sync and
679 	 * save some useful information which will be stored in the
680 	 * core file later.
681 	 */
682 	(void) sprintf((char *)bp, sync_str,
683 	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
684 	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
685 	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
686 	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
687 	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
688 	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
689 	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
690 	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
691 	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
692 	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
693 	prom_interpret(bp, 0, 0, 0, 0, 0);
694 	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
695 }
696 
697 
698 size_t
699 calc_pp_sz(pgcnt_t npages)
700 {
701 
702 	return (npages * sizeof (struct page));
703 }
704 
705 size_t
706 calc_kpmpp_sz(pgcnt_t npages)
707 {
708 
709 	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
710 	kpm_pgsz = 1ull << kpm_pgshft;
711 	kpm_pgoff = kpm_pgsz - 1;
712 	kpmp2pshft = kpm_pgshft - PAGESHIFT;
713 	kpmpnpgs = 1 << kpmp2pshft;
714 
715 	if (kpm_smallpages == 0) {
716 		/*
717 		 * Avoid fragmentation problems in kphysm_init()
718 		 * by allocating for all of physical memory
719 		 */
720 		kpm_npages = ptokpmpr(physinstalled);
721 		return (kpm_npages * sizeof (kpm_page_t));
722 	} else {
723 		kpm_npages = npages;
724 		return (kpm_npages * sizeof (kpm_spage_t));
725 	}
726 }
727 
728 size_t
729 calc_pagehash_sz(pgcnt_t npages)
730 {
731 
732 	/*
733 	 * The page structure hash table size is a power of 2
734 	 * such that the average hash chain length is PAGE_HASHAVELEN.
735 	 */
736 	page_hashsz = npages / PAGE_HASHAVELEN;
737 	page_hashsz = 1 << highbit(page_hashsz);
738 	return (page_hashsz * sizeof (struct page *));
739 }
740 
741 void
742 alloc_kmem64(caddr_t base, caddr_t end)
743 {
744 	int i;
745 	caddr_t aligned_end = NULL;
746 
747 	/*
748 	 * Make one large memory alloc after figuring out the 64-bit size. This
749 	 * will enable use of the largest page size appropriate for the system
750 	 * architecture.
751 	 */
752 	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
753 	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
754 	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
755 		size_t alloc_size, alignsize;
756 #if !defined(C_OBP)
757 		unsigned long long pa;
758 #endif	/* !C_OBP */
759 
760 		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
761 			continue;
762 		alignsize = TTEBYTES(i);
763 		kmem64_szc = i;
764 
765 		/* limit page size for small memory */
766 		if (mmu_btop(alignsize) > (npages >> 2))
767 			continue;
768 
769 		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
770 		alloc_size = aligned_end - base;
771 #if !defined(C_OBP)
772 		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
773 			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
774 				kmem64_pabase = pa;
775 				kmem64_aligned_end = aligned_end;
776 				install_kmem64_tte();
777 				break;
778 			} else {
779 				prom_free_phys(alloc_size, pa);
780 			}
781 		}
782 #else	/* !C_OBP */
783 		if (prom_alloc(base, alloc_size, alignsize) == base) {
784 			kmem64_pabase = va_to_pa(kmem64_base);
785 			kmem64_aligned_end = aligned_end;
786 			break;
787 		}
788 #endif	/* !C_OBP */
789 		if (i == TTE8K) {
790 			prom_panic("kmem64 allocation failure");
791 		}
792 	}
793 	ASSERT(aligned_end != NULL);
794 }
795 
796 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
797 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
798 
799 #define	IVSIZE	roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \
800 			(MAX_RSVD_IV * sizeof (intr_vec_t)) + \
801 			(MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE)
802 
803 #if !defined(C_OBP)
804 /*
805  * Install a temporary tte handler in OBP for kmem64 area.
806  *
807  * We map kmem64 area with large pages before the trap table is taken
808  * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
809  * the same area. Duplicate tlb entries with different page sizes
810  * cause unpredicatble behavior.  To avoid this, we don't create
811  * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
812  * OBP).  Instead, we manage translations with a temporary va>tte-data
813  * handler (kmem64-tte).  This handler is replaced by unix-tte when
814  * the trap table is taken over.
815  *
816  * The temporary handler knows the physical address of the kmem64
817  * area. It uses the prom's pgmap@ Forth word for other addresses.
818  *
819  * We have to use BOP_ALLOC() method for C-OBP platforms because
820  * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
821  * sun4u platforms. On sun4u we flush tlb after trap table is taken
822  * over if we use large pages for kernel heap and kmem64. Since sun4u
823  * prom (unlike sun4v) calls va>tte-data first for client address
824  * translation prom's ttes for kmem64 can't get into TLB even if we
825  * later switch to prom's trap table again. C-OBP uses 4M pages for
826  * client mappings when possible so on all platforms we get the
827  * benefit from large mappings for kmem64 area immediately during
828  * boot.
829  *
830  * pseudo code:
831  * if (context != 0) {
832  * 	return false
833  * } else if (miss_va in range[kmem64_base, kmem64_end)) {
834  *	tte = tte_template +
835  *		(((miss_va & pagemask) - kmem64_base));
836  *	return tte, true
837  * } else {
838  *	return pgmap@ result
839  * }
840  */
841 char kmem64_obp_str[] =
842 	"h# %lx constant kmem64-base "
843 	"h# %lx constant kmem64-end "
844 	"h# %lx constant kmem64-pagemask "
845 	"h# %lx constant kmem64-template "
846 
847 	": kmem64-tte ( addr cnum -- false | tte-data true ) "
848 	"    if                                       ( addr ) "
849 	"       drop false exit then                  ( false ) "
850 	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
851 	"	kmem64-pagemask and                   ( addr' ) "
852 	"	kmem64-base -                         ( addr' ) "
853 	"	kmem64-template +                     ( tte ) "
854 	"	true                                  ( tte true ) "
855 	"    else                                     ( addr ) "
856 	"	pgmap@                                ( tte ) "
857 	"       dup 0< if true else drop false then   ( tte true  |  false ) "
858 	"    then                                     ( tte true  |  false ) "
859 	"; "
860 
861 	"' kmem64-tte is va>tte-data "
862 ;
863 
864 static void
865 install_kmem64_tte()
866 {
867 	char b[sizeof (kmem64_obp_str) + (4 * 16)];
868 	tte_t tte;
869 
870 	PRM_DEBUG(kmem64_pabase);
871 	PRM_DEBUG(kmem64_szc);
872 	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
873 	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
874 	PRM_DEBUG(tte.ll);
875 	(void) sprintf(b, kmem64_obp_str,
876 	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
877 	ASSERT(strlen(b) < sizeof (b));
878 	prom_interpret(b, 0, 0, 0, 0, 0);
879 }
880 #endif	/* !C_OBP */
881 
882 /*
883  * As OBP takes up some RAM when the system boots, pages will already be "lost"
884  * to the system and reflected in npages by the time we see it.
885  *
886  * We only want to allocate kernel structures in the 64-bit virtual address
887  * space on systems with enough RAM to make the overhead of keeping track of
888  * an extra kernel memory segment worthwhile.
889  *
890  * Since OBP has already performed its memory allocations by this point, if we
891  * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
892  * memory in the 64-bit virtual address space; otherwise keep allocations
893  * contiguous with we've mapped so far in the 32-bit virtual address space.
894  */
895 #define	MINMOVE_RAM_MB	((size_t)1900)
896 #define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
897 #define	BYTES_TO_MB(b) ((b) / 1048576ul)
898 
899 pgcnt_t	tune_npages = (pgcnt_t)
900 	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
901 
902 #pragma weak page_set_colorequiv_arr_cpu
903 extern void page_set_colorequiv_arr_cpu(void);
904 extern void page_set_colorequiv_arr(void);
905 
906 static pgcnt_t ramdisk_npages;
907 static struct memlist *old_phys_avail;
908 
909 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
910 
911 static void
912 startup_memlist(void)
913 {
914 	size_t hmehash_sz, pagelist_sz, tt_sz;
915 	size_t psetable_sz;
916 	caddr_t alloc_base;
917 	caddr_t memspace;
918 	struct memlist *cur;
919 	size_t syslimit = (size_t)SYSLIMIT;
920 	size_t sysbase = (size_t)SYSBASE;
921 
922 	/*
923 	 * Initialize enough of the system to allow kmem_alloc to work by
924 	 * calling boot to allocate its memory until the time that
925 	 * kvm_init is completed.  The page structs are allocated after
926 	 * rounding up end to the nearest page boundary; the memsegs are
927 	 * initialized and the space they use comes from the kernel heap.
928 	 * With appropriate initialization, they can be reallocated later
929 	 * to a size appropriate for the machine's configuration.
930 	 *
931 	 * At this point, memory is allocated for things that will never
932 	 * need to be freed, this used to be "valloced".  This allows a
933 	 * savings as the pages don't need page structures to describe
934 	 * them because them will not be managed by the vm system.
935 	 */
936 
937 	/*
938 	 * We're loaded by boot with the following configuration (as
939 	 * specified in the sun4u/conf/Mapfile):
940 	 *
941 	 * 	text:		4 MB chunk aligned on a 4MB boundary
942 	 * 	data & bss:	4 MB chunk aligned on a 4MB boundary
943 	 *
944 	 * These two chunks will eventually be mapped by 2 locked 4MB
945 	 * ttes and will represent the nucleus of the kernel.  This gives
946 	 * us some free space that is already allocated, some or all of
947 	 * which is made available to kernel module text.
948 	 *
949 	 * The free space in the data-bss chunk is used for nucleus
950 	 * allocatable data structures and we reserve it using the
951 	 * nalloc_base and nalloc_end variables.  This space is currently
952 	 * being used for hat data structures required for tlb miss
953 	 * handling operations.  We align nalloc_base to a l2 cache
954 	 * linesize because this is the line size the hardware uses to
955 	 * maintain cache coherency.
956 	 * 512K is carved out for module data.
957 	 */
958 
959 	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
960 	e_moddata = moddata + MODDATA;
961 	nalloc_base = e_moddata;
962 
963 	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
964 	valloc_base = nalloc_base;
965 
966 	/*
967 	 * Calculate the start of the data segment.
968 	 */
969 	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
970 		prom_panic("nucleus data overflow");
971 
972 	PRM_DEBUG(moddata);
973 	PRM_DEBUG(nalloc_base);
974 	PRM_DEBUG(nalloc_end);
975 
976 	/*
977 	 * Remember any slop after e_text so we can give it to the modules.
978 	 */
979 	PRM_DEBUG(e_text);
980 	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
981 	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
982 		prom_panic("nucleus text overflow");
983 	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
984 	    modtext;
985 	PRM_DEBUG(modtext);
986 	PRM_DEBUG(modtext_sz);
987 
988 	init_boot_memlists();
989 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
990 	    &boot_physavail, &boot_physavail_len,
991 	    &boot_virtavail, &boot_virtavail_len);
992 
993 	/*
994 	 * Remember what the physically available highest page is
995 	 * so that dumpsys works properly, and find out how much
996 	 * memory is installed.
997 	 */
998 	installed_top_size_memlist_array(boot_physinstalled,
999 	    boot_physinstalled_len, &physmax, &physinstalled);
1000 	PRM_DEBUG(physinstalled);
1001 	PRM_DEBUG(physmax);
1002 
1003 	/* Fill out memory nodes config structure */
1004 	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1005 
1006 	/*
1007 	 * npages is the maximum of available physical memory possible.
1008 	 * (ie. it will never be more than this)
1009 	 *
1010 	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1011 	 * using phys_avail will underestimate what will end up being freed.
1012 	 * A better initial guess is just total memory minus the kernel text
1013 	 */
1014 	npages = physinstalled - btop(MMU_PAGESIZE4M);
1015 
1016 	/*
1017 	 * First allocate things that can go in the nucleus data page
1018 	 * (fault status, TSBs, dmv, CPUs)
1019 	 */
1020 	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1021 
1022 	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1023 		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1024 
1025 	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1026 		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1027 
1028 	if (ndata_alloc_dmv(&ndata) != 0)
1029 		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1030 
1031 	if (ndata_alloc_page_mutexs(&ndata) != 0)
1032 		cmn_err(CE_PANIC,
1033 		    "no more nucleus memory after page free lists alloc");
1034 
1035 	if (ndata_alloc_hat(&ndata, npages) != 0)
1036 		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1037 
1038 	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1039 		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1040 
1041 	/*
1042 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1043 	 *
1044 	 * There are comments all over the SFMMU code warning of dire
1045 	 * consequences if the TSBs are moved out of 32-bit space.  This
1046 	 * is largely because the asm code uses "sethi %hi(addr)"-type
1047 	 * instructions which will not provide the expected result if the
1048 	 * address is a 64-bit one.
1049 	 *
1050 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1051 	 */
1052 	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1053 	PRM_DEBUG(alloc_base);
1054 
1055 	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1056 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1057 	PRM_DEBUG(alloc_base);
1058 
1059 	/*
1060 	 * Allocate IOMMU TSB array.  We do this here so that the physical
1061 	 * memory gets deducted from the PROM's physical memory list.
1062 	 */
1063 	alloc_base = iommu_tsb_init(alloc_base);
1064 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1065 	PRM_DEBUG(alloc_base);
1066 
1067 	/*
1068 	 * Allow for an early allocation of physically contiguous memory.
1069 	 */
1070 	alloc_base = contig_mem_prealloc(alloc_base, npages);
1071 
1072 	/*
1073 	 * Platforms like Starcat and OPL need special structures assigned in
1074 	 * 32-bit virtual address space because their probing routines execute
1075 	 * FCode, and FCode can't handle 64-bit virtual addresses...
1076 	 */
1077 	if (&plat_startup_memlist) {
1078 		alloc_base = plat_startup_memlist(alloc_base);
1079 		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1080 		    ecache_alignsize);
1081 		PRM_DEBUG(alloc_base);
1082 	}
1083 
1084 	/*
1085 	 * Save off where the contiguous allocations to date have ended
1086 	 * in econtig32.
1087 	 */
1088 	econtig32 = alloc_base;
1089 	PRM_DEBUG(econtig32);
1090 	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1091 		cmn_err(CE_PANIC, "econtig32 too big");
1092 
1093 	pp_sz = calc_pp_sz(npages);
1094 	PRM_DEBUG(pp_sz);
1095 	if (kpm_enable) {
1096 		kpm_pp_sz = calc_kpmpp_sz(npages);
1097 		PRM_DEBUG(kpm_pp_sz);
1098 	}
1099 
1100 	hmehash_sz = calc_hmehash_sz(npages);
1101 	PRM_DEBUG(hmehash_sz);
1102 
1103 	pagehash_sz = calc_pagehash_sz(npages);
1104 	PRM_DEBUG(pagehash_sz);
1105 
1106 	pagelist_sz = calc_free_pagelist_sz();
1107 	PRM_DEBUG(pagelist_sz);
1108 
1109 #ifdef	TRAPTRACE
1110 	tt_sz = calc_traptrace_sz();
1111 	PRM_DEBUG(tt_sz);
1112 #else
1113 	tt_sz = 0;
1114 #endif	/* TRAPTRACE */
1115 
1116 	/*
1117 	 * Place the array that protects pp->p_selock in the kmem64 wad.
1118 	 */
1119 	pse_shift = size_pse_array(physmem, max_ncpus);
1120 	PRM_DEBUG(pse_shift);
1121 	pse_table_size = 1 << pse_shift;
1122 	PRM_DEBUG(pse_table_size);
1123 	psetable_sz = roundup(
1124 	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1125 	PRM_DEBUG(psetable_sz);
1126 
1127 	/*
1128 	 * Now allocate the whole wad
1129 	 */
1130 	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1131 	    pagelist_sz + tt_sz + psetable_sz;
1132 	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1133 	kmem64_base = (caddr_t)syslimit;
1134 	kmem64_end = kmem64_base + kmem64_sz;
1135 	alloc_kmem64(kmem64_base, kmem64_end);
1136 	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1137 		cmn_err(CE_PANIC, "not enough kmem64 space");
1138 	PRM_DEBUG(kmem64_base);
1139 	PRM_DEBUG(kmem64_end);
1140 	PRM_DEBUG(kmem64_aligned_end);
1141 
1142 	/*
1143 	 * ... and divy it up
1144 	 */
1145 	alloc_base = kmem64_base;
1146 
1147 	if (kpm_smallpages == 0) {
1148 		npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1149 	} else {
1150 		npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1151 		    sizeof (kpm_spage_t));
1152 	}
1153 
1154 	pp_base = (page_t *)alloc_base;
1155 	pp_sz = npages * sizeof (struct page);
1156 	alloc_base += pp_sz;
1157 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1158 	PRM_DEBUG(pp_base);
1159 	PRM_DEBUG(npages);
1160 
1161 	if (kpm_enable) {
1162 		kpm_pp_base = alloc_base;
1163 		if (kpm_smallpages == 0) {
1164 			/* kpm_npages based on physinstalled, don't reset */
1165 			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1166 		} else {
1167 			kpm_npages = ptokpmpr(npages);
1168 			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1169 		}
1170 		alloc_base += kpm_pp_sz;
1171 		alloc_base =
1172 		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1173 		PRM_DEBUG(kpm_pp_base);
1174 	}
1175 
1176 	alloc_base = alloc_hmehash(alloc_base);
1177 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1178 	PRM_DEBUG(alloc_base);
1179 
1180 	page_hash = (page_t **)alloc_base;
1181 	alloc_base += pagehash_sz;
1182 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1183 	PRM_DEBUG(page_hash);
1184 
1185 	alloc_base = alloc_page_freelists(alloc_base);
1186 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1187 	PRM_DEBUG(alloc_base);
1188 
1189 #ifdef	TRAPTRACE
1190 	ttrace_buf = alloc_base;
1191 	alloc_base += tt_sz;
1192 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1193 	PRM_DEBUG(alloc_base);
1194 #endif	/* TRAPTRACE */
1195 
1196 	pse_mutex = (pad_mutex_t *)alloc_base;
1197 	alloc_base += psetable_sz;
1198 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1199 	PRM_DEBUG(alloc_base);
1200 
1201 	/* adjust kmem64_end to what we really allocated */
1202 	kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1203 	kmem64_sz = kmem64_end - kmem64_base;
1204 
1205 	if (&ecache_init_scrub_flush_area) {
1206 		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1207 		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1208 	}
1209 
1210 	/*
1211 	 * If physmem is patched to be non-zero, use it instead of
1212 	 * the monitor value unless physmem is larger than the total
1213 	 * amount of memory on hand.
1214 	 */
1215 	if (physmem == 0 || physmem > npages)
1216 		physmem = npages;
1217 
1218 	/*
1219 	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1220 	 * is mounted as root. This memory is held down by OBP and unlike
1221 	 * the stub boot_archive is never released.
1222 	 *
1223 	 * In order to get things sized correctly on lower memory
1224 	 * machines (where the memory used by the ramdisk represents
1225 	 * a significant portion of memory), physmem is adjusted.
1226 	 *
1227 	 * This is done by subtracting the ramdisk_size which is set
1228 	 * to the size of the ramdisk (in Kb) in /etc/system at the
1229 	 * time the miniroot archive is constructed.
1230 	 */
1231 	if (root_is_ramdisk == B_TRUE) {
1232 		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1233 		physmem -= ramdisk_npages;
1234 	}
1235 
1236 	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1237 		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1238 
1239 	/*
1240 	 * Allocate space for the interrupt vector table.
1241 	 */
1242 	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1243 	if (memspace != (caddr_t)intr_vec_table)
1244 		prom_panic("interrupt vector table allocation failure");
1245 
1246 	/*
1247 	 * Between now and when we finish copying in the memory lists,
1248 	 * allocations happen so the space gets fragmented and the
1249 	 * lists longer.  Leave enough space for lists twice as
1250 	 * long as we have now; then roundup to a pagesize.
1251 	 */
1252 	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1253 	    prom_phys_avail_len() + prom_virt_avail_len());
1254 	memlist_sz *= 2;
1255 	memlist_sz = roundup(memlist_sz, PAGESIZE);
1256 	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1257 	if (memspace == NULL)
1258 		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1259 
1260 	memlist = (struct memlist *)memspace;
1261 	memlist_end = (char *)memspace + memlist_sz;
1262 	PRM_DEBUG(memlist);
1263 	PRM_DEBUG(memlist_end);
1264 
1265 	PRM_DEBUG(sysbase);
1266 	PRM_DEBUG(syslimit);
1267 	kernelheap_init((void *)sysbase, (void *)syslimit,
1268 	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1269 
1270 	/*
1271 	 * Take the most current snapshot we can by calling mem-update.
1272 	 */
1273 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1274 	    &boot_physavail, &boot_physavail_len,
1275 	    &boot_virtavail, &boot_virtavail_len);
1276 
1277 	/*
1278 	 * Remove the space used by prom_alloc from the kernel heap
1279 	 * plus the area actually used by the OBP (if any)
1280 	 * ignoring virtual addresses in virt_avail, above syslimit.
1281 	 */
1282 	virt_avail = memlist;
1283 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1284 
1285 	for (cur = virt_avail; cur->next; cur = cur->next) {
1286 		uint64_t range_base, range_size;
1287 
1288 		if ((range_base = cur->address + cur->size) < (uint64_t)sysbase)
1289 			continue;
1290 		if (range_base >= (uint64_t)syslimit)
1291 			break;
1292 		/*
1293 		 * Limit the range to end at syslimit.
1294 		 */
1295 		range_size = MIN(cur->next->address,
1296 		    (uint64_t)syslimit) - range_base;
1297 		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1298 		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1299 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1300 	}
1301 
1302 	phys_avail = memlist;
1303 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1304 
1305 	/*
1306 	 * Add any extra memory at the end of the ndata region if there's at
1307 	 * least a page to add.  There might be a few more pages available in
1308 	 * the middle of the ndata region, but for now they are ignored.
1309 	 */
1310 	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1311 	if (nalloc_base == NULL)
1312 		nalloc_base = nalloc_end;
1313 	ndata_remain_sz = nalloc_end - nalloc_base;
1314 
1315 	/*
1316 	 * Copy physinstalled list into kernel space.
1317 	 */
1318 	phys_install = memlist;
1319 	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1320 
1321 	/*
1322 	 * Create list of physical addrs we don't need pp's for:
1323 	 * kernel text 4M page
1324 	 * kernel data 4M page - ndata_remain_sz
1325 	 * kmem64 pages
1326 	 *
1327 	 * NB if adding any pages here, make sure no kpm page
1328 	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1329 	 */
1330 	nopp_list = memlist;
1331 	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1332 	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1333 	    &memlist, &nopp_list);
1334 	memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1335 
1336 	if ((caddr_t)memlist > (memspace + memlist_sz))
1337 		prom_panic("memlist overflow");
1338 
1339 	/*
1340 	 * Initialize the page structures from the memory lists.
1341 	 */
1342 	kphysm_init();
1343 
1344 	availrmem_initial = availrmem = freemem;
1345 	PRM_DEBUG(availrmem);
1346 
1347 	/*
1348 	 * Some of the locks depend on page_hashsz being set!
1349 	 * kmem_init() depends on this; so, keep it here.
1350 	 */
1351 	page_lock_init();
1352 
1353 	/*
1354 	 * Initialize kernel memory allocator.
1355 	 */
1356 	kmem_init();
1357 
1358 	/*
1359 	 * Factor in colorequiv to check additional 'equivalent' bins
1360 	 */
1361 	if (&page_set_colorequiv_arr_cpu != NULL)
1362 		page_set_colorequiv_arr_cpu();
1363 	else
1364 		page_set_colorequiv_arr();
1365 
1366 	/*
1367 	 * Initialize bp_mapin().
1368 	 */
1369 	bp_init(shm_alignment, HAT_STRICTORDER);
1370 
1371 	/*
1372 	 * Reserve space for panicbuf, intr_vec_table, reserved interrupt
1373 	 * vector data structures and MPO mblock structs from the 32-bit heap.
1374 	 */
1375 	(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
1376 	    panicbuf, panicbuf + PANICBUFSIZE,
1377 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1378 
1379 	(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
1380 	    intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
1381 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1382 
1383 	if (mpo_heap32_bufsz > (size_t)0) {
1384 		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1385 		    PAGESIZE, 0, 0, mpo_heap32_buf,
1386 		    mpo_heap32_buf + mpo_heap32_bufsz,
1387 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1388 	}
1389 	mem_config_init();
1390 }
1391 
1392 static void
1393 startup_modules(void)
1394 {
1395 	int nhblk1, nhblk8;
1396 	size_t  nhblksz;
1397 	pgcnt_t pages_per_hblk;
1398 	size_t hme8blk_sz, hme1blk_sz;
1399 
1400 	/*
1401 	 * Let the platforms have a chance to change default
1402 	 * values before reading system file.
1403 	 */
1404 	if (&set_platform_defaults)
1405 		set_platform_defaults();
1406 
1407 	/*
1408 	 * Calculate default settings of system parameters based upon
1409 	 * maxusers, yet allow to be overridden via the /etc/system file.
1410 	 */
1411 	param_calc(0);
1412 
1413 	mod_setup();
1414 
1415 	/*
1416 	 * If this is a positron, complain and halt.
1417 	 */
1418 	if (&iam_positron && iam_positron()) {
1419 		cmn_err(CE_WARN, "This hardware platform is not supported"
1420 		    " by this release of Solaris.\n");
1421 #ifdef DEBUG
1422 		prom_enter_mon();	/* Type 'go' to resume */
1423 		cmn_err(CE_WARN, "Booting an unsupported platform.\n");
1424 		cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
1425 
1426 #else /* DEBUG */
1427 		halt(0);
1428 #endif /* DEBUG */
1429 	}
1430 
1431 	/*
1432 	 * If we are running firmware that isn't 64-bit ready
1433 	 * then complain and halt.
1434 	 */
1435 	do_prom_version_check();
1436 
1437 	/*
1438 	 * Initialize system parameters
1439 	 */
1440 	param_init();
1441 
1442 	/*
1443 	 * maxmem is the amount of physical memory we're playing with.
1444 	 */
1445 	maxmem = physmem;
1446 
1447 	/* Set segkp limits. */
1448 	ncbase = kdi_segdebugbase;
1449 	ncend = kdi_segdebugbase;
1450 
1451 	/*
1452 	 * Initialize the hat layer.
1453 	 */
1454 	hat_init();
1455 
1456 	/*
1457 	 * Initialize segment management stuff.
1458 	 */
1459 	seg_init();
1460 
1461 	/*
1462 	 * Create the va>tte handler, so the prom can understand
1463 	 * kernel translations.  The handler is installed later, just
1464 	 * as we are about to take over the trap table from the prom.
1465 	 */
1466 	create_va_to_tte();
1467 
1468 	/*
1469 	 * Load the forthdebugger (optional)
1470 	 */
1471 	forthdebug_init();
1472 
1473 	/*
1474 	 * Create OBP node for console input callbacks
1475 	 * if it is needed.
1476 	 */
1477 	startup_create_io_node();
1478 
1479 	if (modloadonly("fs", "specfs") == -1)
1480 		halt("Can't load specfs");
1481 
1482 	if (modloadonly("fs", "devfs") == -1)
1483 		halt("Can't load devfs");
1484 
1485 	if (modloadonly("misc", "swapgeneric") == -1)
1486 		halt("Can't load swapgeneric");
1487 
1488 	(void) modloadonly("sys", "lbl_edition");
1489 
1490 	dispinit();
1491 
1492 	/*
1493 	 * Infer meanings to the members of the idprom buffer.
1494 	 */
1495 	parse_idprom();
1496 
1497 	/* Read cluster configuration data. */
1498 	clconf_init();
1499 
1500 	setup_ddi();
1501 
1502 	/*
1503 	 * Lets take this opportunity to load the root device.
1504 	 */
1505 	if (loadrootmodules() != 0)
1506 		debug_enter("Can't load the root filesystem");
1507 
1508 	/*
1509 	 * Load tod driver module for the tod part found on this system.
1510 	 * Recompute the cpu frequency/delays based on tod as tod part
1511 	 * tends to keep time more accurately.
1512 	 */
1513 	if (&load_tod_module)
1514 		load_tod_module();
1515 
1516 	/*
1517 	 * Allow platforms to load modules which might
1518 	 * be needed after bootops are gone.
1519 	 */
1520 	if (&load_platform_modules)
1521 		load_platform_modules();
1522 
1523 	setcpudelay();
1524 
1525 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1526 	    &boot_physavail, &boot_physavail_len,
1527 	    &boot_virtavail, &boot_virtavail_len);
1528 
1529 	/*
1530 	 * Calculation and allocation of hmeblks needed to remap
1531 	 * the memory allocated by PROM till now.
1532 	 * Overestimate the number of hblk1 elements by assuming
1533 	 * worst case of TTE64K mappings.
1534 	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1535 	 */
1536 	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1537 	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1538 	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1539 	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1540 
1541 	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1542 
1543 	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1544 	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1545 	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1546 
1547 	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1548 
1549 	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1550 	nhblk8 = 0;
1551 	while (bop_alloc_pages > 1) {
1552 		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1553 		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1554 		bop_alloc_pages *= hme8blk_sz;
1555 		bop_alloc_pages = btopr(bop_alloc_pages);
1556 	}
1557 	nhblk8 += 2;
1558 
1559 	/*
1560 	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1561 	 * boundary, the number of hblk8's needed to map the entries in the
1562 	 * boot_virtavail list needs to be adjusted to take this into
1563 	 * consideration.  Thus, we need to add additional hblk8's since it
1564 	 * is possible that an hblk8 will not have all 8 slots used due to
1565 	 * alignment constraints.  Since there were boot_virtavail_len entries
1566 	 * in that list, we need to add that many hblk8's to the number
1567 	 * already calculated to make sure we don't underestimate.
1568 	 */
1569 	nhblk8 += boot_virtavail_len;
1570 	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1571 
1572 	/* Allocate in pagesize chunks */
1573 	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1574 	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1575 	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1576 }
1577 
1578 static void
1579 startup_bop_gone(void)
1580 {
1581 
1582 	/*
1583 	 * Destroy the MD initialized at startup
1584 	 * The startup initializes the MD framework
1585 	 * using prom and BOP alloc free it now.
1586 	 */
1587 	mach_descrip_startup_fini();
1588 
1589 	/*
1590 	 * We're done with prom allocations.
1591 	 */
1592 	bop_fini();
1593 
1594 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1595 	    &boot_physavail, &boot_physavail_len,
1596 	    &boot_virtavail, &boot_virtavail_len);
1597 
1598 	/*
1599 	 * setup physically contiguous area twice as large as the ecache.
1600 	 * this is used while doing displacement flush of ecaches
1601 	 */
1602 	if (&ecache_flush_address) {
1603 		ecache_flushaddr = ecache_flush_address();
1604 		if (ecache_flushaddr == (uint64_t)-1) {
1605 			cmn_err(CE_PANIC,
1606 			    "startup: no memory to set ecache_flushaddr");
1607 		}
1608 	}
1609 
1610 	/*
1611 	 * Virtual available next.
1612 	 */
1613 	ASSERT(virt_avail != NULL);
1614 	memlist_free_list(virt_avail);
1615 	virt_avail = memlist;
1616 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1617 
1618 }
1619 
1620 
1621 /*
1622  * startup_fixup_physavail - called from mach_sfmmu.c after the final
1623  * allocations have been performed.  We can't call it in startup_bop_gone
1624  * since later operations can cause obp to allocate more memory.
1625  */
1626 void
1627 startup_fixup_physavail(void)
1628 {
1629 	struct memlist *cur;
1630 	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1631 
1632 	PRM_DEBUG(kmem64_overmap_size);
1633 
1634 	/*
1635 	 * take the most current snapshot we can by calling mem-update
1636 	 */
1637 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1638 	    &boot_physavail, &boot_physavail_len,
1639 	    &boot_virtavail, &boot_virtavail_len);
1640 
1641 	/*
1642 	 * Copy phys_avail list, again.
1643 	 * Both the kernel/boot and the prom have been allocating
1644 	 * from the original list we copied earlier.
1645 	 */
1646 	cur = memlist;
1647 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1648 
1649 	/*
1650 	 * Add any unused kmem64 memory from overmapped page
1651 	 * (Note: va_to_pa does not work for kmem64_end)
1652 	 */
1653 	if (kmem64_overmap_size) {
1654 		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1655 		    kmem64_overmap_size, &memlist, &cur);
1656 	}
1657 
1658 	/*
1659 	 * Add any extra memory after e_data we added to the phys_avail list
1660 	 * back to the old list.
1661 	 */
1662 	if (ndata_remain_sz >= MMU_PAGESIZE)
1663 		memlist_add(va_to_pa(nalloc_base),
1664 		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1665 
1666 	/*
1667 	 * There isn't any bounds checking on the memlist area
1668 	 * so ensure it hasn't overgrown.
1669 	 */
1670 	if ((caddr_t)memlist > (caddr_t)memlist_end)
1671 		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1672 
1673 	/*
1674 	 * The kernel removes the pages that were allocated for it from
1675 	 * the freelist, but we now have to find any -extra- pages that
1676 	 * the prom has allocated for it's own book-keeping, and remove
1677 	 * them from the freelist too. sigh.
1678 	 */
1679 	sync_memlists(phys_avail, cur);
1680 
1681 	ASSERT(phys_avail != NULL);
1682 
1683 	old_phys_avail = phys_avail;
1684 	phys_avail = cur;
1685 }
1686 
1687 void
1688 update_kcage_ranges(uint64_t addr, uint64_t len)
1689 {
1690 	pfn_t base = btop(addr);
1691 	pgcnt_t num = btop(len);
1692 	int rv;
1693 
1694 	rv = kcage_range_add(base, num, kcage_startup_dir);
1695 
1696 	if (rv == ENOMEM) {
1697 		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1698 		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1699 	} else if (rv != 0) {
1700 		/* catch this in debug kernels */
1701 		ASSERT(0);
1702 
1703 		cmn_err(CE_WARN, "unexpected kcage_range_add"
1704 		    " return value %d", rv);
1705 	}
1706 }
1707 
1708 static void
1709 startup_vm(void)
1710 {
1711 	size_t	i;
1712 	struct segmap_crargs a;
1713 	struct segkpm_crargs b;
1714 
1715 	uint64_t avmem;
1716 	caddr_t va;
1717 	pgcnt_t	max_phys_segkp;
1718 	int	mnode;
1719 
1720 	extern int use_brk_lpg, use_stk_lpg;
1721 
1722 	/*
1723 	 * get prom's mappings, create hments for them and switch
1724 	 * to the kernel context.
1725 	 */
1726 	hat_kern_setup();
1727 
1728 	/*
1729 	 * Take over trap table
1730 	 */
1731 	setup_trap_table();
1732 
1733 	/*
1734 	 * Install the va>tte handler, so that the prom can handle
1735 	 * misses and understand the kernel table layout in case
1736 	 * we need call into the prom.
1737 	 */
1738 	install_va_to_tte();
1739 
1740 	/*
1741 	 * Set a flag to indicate that the tba has been taken over.
1742 	 */
1743 	tba_taken_over = 1;
1744 
1745 	/* initialize MMU primary context register */
1746 	mmu_init_kcontext();
1747 
1748 	/*
1749 	 * The boot cpu can now take interrupts, x-calls, x-traps
1750 	 */
1751 	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1752 	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1753 
1754 	/*
1755 	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1756 	 */
1757 	tbr_wr_addr_inited = 1;
1758 
1759 	/*
1760 	 * Initialize VM system, and map kernel address space.
1761 	 */
1762 	kvm_init();
1763 
1764 	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1765 	if (kernel_cage_enable) {
1766 		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1767 	}
1768 	memlist_free_list(old_phys_avail);
1769 
1770 	/*
1771 	 * If the following is true, someone has patched
1772 	 * phsymem to be less than the number of pages that
1773 	 * the system actually has.  Remove pages until system
1774 	 * memory is limited to the requested amount.  Since we
1775 	 * have allocated page structures for all pages, we
1776 	 * correct the amount of memory we want to remove
1777 	 * by the size of the memory used to hold page structures
1778 	 * for the non-used pages.
1779 	 */
1780 	if (physmem + ramdisk_npages < npages) {
1781 		pgcnt_t diff, off;
1782 		struct page *pp;
1783 		struct seg kseg;
1784 
1785 		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1786 
1787 		off = 0;
1788 		diff = npages - (physmem + ramdisk_npages);
1789 		diff -= mmu_btopr(diff * sizeof (struct page));
1790 		kseg.s_as = &kas;
1791 		while (diff--) {
1792 			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1793 			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1794 			    &kseg, (caddr_t)off);
1795 			if (pp == NULL)
1796 				cmn_err(CE_PANIC, "limited physmem too much!");
1797 			page_io_unlock(pp);
1798 			page_downgrade(pp);
1799 			availrmem--;
1800 			off += MMU_PAGESIZE;
1801 		}
1802 	}
1803 
1804 	/*
1805 	 * When printing memory, show the total as physmem less
1806 	 * that stolen by a debugger.
1807 	 */
1808 	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1809 	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1810 	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1811 
1812 	avmem = (uint64_t)freemem << PAGESHIFT;
1813 	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1814 
1815 	/*
1816 	 * For small memory systems disable automatic large pages.
1817 	 */
1818 	if (physmem < privm_lpg_min_physmem) {
1819 		use_brk_lpg = 0;
1820 		use_stk_lpg = 0;
1821 	}
1822 
1823 	/*
1824 	 * Perform platform specific freelist processing
1825 	 */
1826 	if (&plat_freelist_process) {
1827 		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1828 			if (mem_node_config[mnode].exists)
1829 				plat_freelist_process(mnode);
1830 	}
1831 
1832 	/*
1833 	 * Initialize the segkp segment type.  We position it
1834 	 * after the configured tables and buffers (whose end
1835 	 * is given by econtig) and before V_WKBASE_ADDR.
1836 	 * Also in this area is segkmap (size SEGMAPSIZE).
1837 	 */
1838 
1839 	/* XXX - cache alignment? */
1840 	va = (caddr_t)SEGKPBASE;
1841 	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1842 
1843 	max_phys_segkp = (physmem * 2);
1844 
1845 	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1846 		segkpsize = btop(SEGKPDEFSIZE);
1847 		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1848 		    "segkpsize has been reset to %ld pages", segkpsize);
1849 	}
1850 
1851 	i = ptob(MIN(segkpsize, max_phys_segkp));
1852 
1853 	rw_enter(&kas.a_lock, RW_WRITER);
1854 	if (seg_attach(&kas, va, i, segkp) < 0)
1855 		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1856 	if (segkp_create(segkp) != 0)
1857 		cmn_err(CE_PANIC, "startup: segkp_create failed");
1858 	rw_exit(&kas.a_lock);
1859 
1860 	/*
1861 	 * kpm segment
1862 	 */
1863 	segmap_kpm = kpm_enable &&
1864 	    segmap_kpm && PAGESIZE == MAXBSIZE;
1865 
1866 	if (kpm_enable) {
1867 		rw_enter(&kas.a_lock, RW_WRITER);
1868 
1869 		/*
1870 		 * The segkpm virtual range range is larger than the
1871 		 * actual physical memory size and also covers gaps in
1872 		 * the physical address range for the following reasons:
1873 		 * . keep conversion between segkpm and physical addresses
1874 		 *   simple, cheap and unambiguous.
1875 		 * . avoid extension/shrink of the the segkpm in case of DR.
1876 		 * . avoid complexity for handling of virtual addressed
1877 		 *   caches, segkpm and the regular mapping scheme must be
1878 		 *   kept in sync wrt. the virtual color of mapped pages.
1879 		 * Any accesses to virtual segkpm ranges not backed by
1880 		 * physical memory will fall through the memseg pfn hash
1881 		 * and will be handled in segkpm_fault.
1882 		 * Additional kpm_size spaces needed for vac alias prevention.
1883 		 */
1884 		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1885 		    segkpm) < 0)
1886 			cmn_err(CE_PANIC, "cannot attach segkpm");
1887 
1888 		b.prot = PROT_READ | PROT_WRITE;
1889 		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1890 
1891 		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1892 			panic("segkpm_create segkpm");
1893 
1894 		rw_exit(&kas.a_lock);
1895 
1896 		mach_kpm_init();
1897 	}
1898 
1899 	if (!segzio_fromheap) {
1900 		size_t size;
1901 		size_t physmem_b = mmu_ptob(physmem);
1902 
1903 		/* size is in bytes, segziosize is in pages */
1904 		if (segziosize == 0) {
1905 			size = physmem_b;
1906 		} else {
1907 			size = mmu_ptob(segziosize);
1908 		}
1909 
1910 		if (size < SEGZIOMINSIZE) {
1911 			size = SEGZIOMINSIZE;
1912 		} else if (size > SEGZIOMAXSIZE) {
1913 			size = SEGZIOMAXSIZE;
1914 			/*
1915 			 * On 64-bit x86, we only have 2TB of KVA.  This exists
1916 			 * for parity with x86.
1917 			 *
1918 			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
1919 			 * doesn't consume all of KVA.  However, if we have a
1920 			 * system that has more thant 512gb of physical memory,
1921 			 * we can actually consume about half of the difference
1922 			 * between 512gb and the rest of the available physical
1923 			 * memory.
1924 			 */
1925 			if (physmem_b > SEGZIOMAXSIZE) {
1926 				size += (physmem_b - SEGZIOMAXSIZE) / 2;
1927 		}
1928 		}
1929 		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
1930 		/* put the base of the ZIO segment after the kpm segment */
1931 		segzio_base = kpm_vbase + (kpm_size * vac_colors);
1932 		PRM_DEBUG(segziosize);
1933 		PRM_DEBUG(segzio_base);
1934 
1935 		/*
1936 		 * On some platforms, kvm_init is called after the kpm
1937 		 * sizes have been determined.  On SPARC, kvm_init is called
1938 		 * before, so we have to attach the kzioseg after kvm is
1939 		 * initialized, otherwise we'll try to allocate from the boot
1940 		 * area since the kernel heap hasn't yet been configured.
1941 		 */
1942 		rw_enter(&kas.a_lock, RW_WRITER);
1943 
1944 		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
1945 		    &kzioseg);
1946 		(void) segkmem_zio_create(&kzioseg);
1947 
1948 		/* create zio area covering new segment */
1949 		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
1950 
1951 		rw_exit(&kas.a_lock);
1952 	}
1953 
1954 
1955 	/*
1956 	 * Now create generic mapping segment.  This mapping
1957 	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
1958 	 * virtual address is greater than the amount of free
1959 	 * memory that is available, then we trim back the
1960 	 * segment size to that amount
1961 	 */
1962 	va = (caddr_t)SEGMAPBASE;
1963 
1964 	/*
1965 	 * 1201049: segkmap base address must be MAXBSIZE aligned
1966 	 */
1967 	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
1968 
1969 	/*
1970 	 * Set size of segmap to percentage of freemem at boot,
1971 	 * but stay within the allowable range
1972 	 * Note we take percentage  before converting from pages
1973 	 * to bytes to avoid an overflow on 32-bit kernels.
1974 	 */
1975 	i = mmu_ptob((freemem * segmap_percent) / 100);
1976 
1977 	if (i < MINMAPSIZE)
1978 		i = MINMAPSIZE;
1979 
1980 	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
1981 		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
1982 
1983 	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
1984 
1985 	rw_enter(&kas.a_lock, RW_WRITER);
1986 	if (seg_attach(&kas, va, i, segkmap) < 0)
1987 		cmn_err(CE_PANIC, "cannot attach segkmap");
1988 
1989 	a.prot = PROT_READ | PROT_WRITE;
1990 	a.shmsize = shm_alignment;
1991 	a.nfreelist = 0;	/* use segmap driver defaults */
1992 
1993 	if (segmap_create(segkmap, (caddr_t)&a) != 0)
1994 		panic("segmap_create segkmap");
1995 	rw_exit(&kas.a_lock);
1996 
1997 	segdev_init();
1998 }
1999 
2000 static void
2001 startup_end(void)
2002 {
2003 	if ((caddr_t)memlist > (caddr_t)memlist_end)
2004 		panic("memlist overflow 2");
2005 	memlist_free_block((caddr_t)memlist,
2006 	    ((caddr_t)memlist_end - (caddr_t)memlist));
2007 	memlist = NULL;
2008 
2009 	/* enable page_relocation since OBP is now done */
2010 	page_relocate_ready = 1;
2011 
2012 	/*
2013 	 * Perform tasks that get done after most of the VM
2014 	 * initialization has been done but before the clock
2015 	 * and other devices get started.
2016 	 */
2017 	kern_setup1();
2018 
2019 	/*
2020 	 * Intialize the VM arenas for allocating physically
2021 	 * contiguus memory chunk for interrupt queues snd
2022 	 * allocate/register boot cpu's queues, if any and
2023 	 * allocate dump buffer for sun4v systems to store
2024 	 * extra crash information during crash dump
2025 	 */
2026 	contig_mem_init();
2027 	mach_descrip_init();
2028 
2029 	if (cpu_intrq_setup(CPU)) {
2030 		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2031 	}
2032 	cpu_intrq_register(CPU);
2033 	mach_htraptrace_setup(CPU->cpu_id);
2034 	mach_htraptrace_configure(CPU->cpu_id);
2035 	mach_dump_buffer_init();
2036 
2037 	/*
2038 	 * Initialize interrupt related stuff
2039 	 */
2040 	cpu_intr_alloc(CPU, NINTR_THREADS);
2041 
2042 	(void) splzs();			/* allow hi clock ints but not zs */
2043 
2044 	/*
2045 	 * Initialize errors.
2046 	 */
2047 	error_init();
2048 
2049 	/*
2050 	 * Note that we may have already used kernel bcopy before this
2051 	 * point - but if you really care about this, adb the use_hw_*
2052 	 * variables to 0 before rebooting.
2053 	 */
2054 	mach_hw_copy_limit();
2055 
2056 	/*
2057 	 * Install the "real" preemption guards before DDI services
2058 	 * are available.
2059 	 */
2060 	(void) prom_set_preprom(kern_preprom);
2061 	(void) prom_set_postprom(kern_postprom);
2062 	CPU->cpu_m.mutex_ready = 1;
2063 
2064 	/*
2065 	 * Initialize segnf (kernel support for non-faulting loads).
2066 	 */
2067 	segnf_init();
2068 
2069 	/*
2070 	 * Configure the root devinfo node.
2071 	 */
2072 	configure();		/* set up devices */
2073 	mach_cpu_halt_idle();
2074 }
2075 
2076 
2077 void
2078 post_startup(void)
2079 {
2080 #ifdef	PTL1_PANIC_DEBUG
2081 	extern void init_ptl1_thread(void);
2082 #endif	/* PTL1_PANIC_DEBUG */
2083 	extern void abort_sequence_init(void);
2084 
2085 	/*
2086 	 * Set the system wide, processor-specific flags to be passed
2087 	 * to userland via the aux vector for performance hints and
2088 	 * instruction set extensions.
2089 	 */
2090 	bind_hwcap();
2091 
2092 	/*
2093 	 * Startup memory scrubber (if any)
2094 	 */
2095 	mach_memscrub();
2096 
2097 	/*
2098 	 * Allocate soft interrupt to handle abort sequence.
2099 	 */
2100 	abort_sequence_init();
2101 
2102 	/*
2103 	 * Configure the rest of the system.
2104 	 * Perform forceloading tasks for /etc/system.
2105 	 */
2106 	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2107 	/*
2108 	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2109 	 * ON4.0: This must be fixed or restated in /etc/systems.
2110 	 */
2111 	(void) modload("fs", "procfs");
2112 
2113 	/* load machine class specific drivers */
2114 	load_mach_drivers();
2115 
2116 	/* load platform specific drivers */
2117 	if (&load_platform_drivers)
2118 		load_platform_drivers();
2119 
2120 	/* load vis simulation module, if we are running w/fpu off */
2121 	if (!fpu_exists) {
2122 		if (modload("misc", "vis") == -1)
2123 			halt("Can't load vis");
2124 	}
2125 
2126 	mach_fpras();
2127 
2128 	maxmem = freemem;
2129 
2130 #ifdef	PTL1_PANIC_DEBUG
2131 	init_ptl1_thread();
2132 #endif	/* PTL1_PANIC_DEBUG */
2133 }
2134 
2135 #ifdef	PTL1_PANIC_DEBUG
2136 int		ptl1_panic_test = 0;
2137 int		ptl1_panic_xc_one_test = 0;
2138 int		ptl1_panic_xc_all_test = 0;
2139 int		ptl1_panic_xt_one_test = 0;
2140 int		ptl1_panic_xt_all_test = 0;
2141 kthread_id_t	ptl1_thread_p = NULL;
2142 kcondvar_t	ptl1_cv;
2143 kmutex_t	ptl1_mutex;
2144 int		ptl1_recurse_count_threshold = 0x40;
2145 int		ptl1_recurse_trap_threshold = 0x3d;
2146 extern void	ptl1_recurse(int, int);
2147 extern void	ptl1_panic_xt(int, int);
2148 
2149 /*
2150  * Called once per second by timeout() to wake up
2151  * the ptl1_panic thread to see if it should cause
2152  * a trap to the ptl1_panic() code.
2153  */
2154 /* ARGSUSED */
2155 static void
2156 ptl1_wakeup(void *arg)
2157 {
2158 	mutex_enter(&ptl1_mutex);
2159 	cv_signal(&ptl1_cv);
2160 	mutex_exit(&ptl1_mutex);
2161 }
2162 
2163 /*
2164  * ptl1_panic cross call function:
2165  *     Needed because xc_one() and xc_some() can pass
2166  *	64 bit args but ptl1_recurse() expects ints.
2167  */
2168 static void
2169 ptl1_panic_xc(void)
2170 {
2171 	ptl1_recurse(ptl1_recurse_count_threshold,
2172 	    ptl1_recurse_trap_threshold);
2173 }
2174 
2175 /*
2176  * The ptl1 thread waits for a global flag to be set
2177  * and uses the recurse thresholds to set the stack depth
2178  * to cause a ptl1_panic() directly via a call to ptl1_recurse
2179  * or indirectly via the cross call and cross trap functions.
2180  *
2181  * This is useful testing stack overflows and normal
2182  * ptl1_panic() states with a know stack frame.
2183  *
2184  * ptl1_recurse() is an asm function in ptl1_panic.s that
2185  * sets the {In, Local, Out, and Global} registers to a
2186  * know state on the stack and just prior to causing a
2187  * test ptl1_panic trap.
2188  */
2189 static void
2190 ptl1_thread(void)
2191 {
2192 	mutex_enter(&ptl1_mutex);
2193 	while (ptl1_thread_p) {
2194 		cpuset_t	other_cpus;
2195 		int		cpu_id;
2196 		int		my_cpu_id;
2197 		int		target_cpu_id;
2198 		int		target_found;
2199 
2200 		if (ptl1_panic_test) {
2201 			ptl1_recurse(ptl1_recurse_count_threshold,
2202 			    ptl1_recurse_trap_threshold);
2203 		}
2204 
2205 		/*
2206 		 * Find potential targets for x-call and x-trap,
2207 		 * if any exist while preempt is disabled we
2208 		 * start a ptl1_panic if requested via a
2209 		 * globals.
2210 		 */
2211 		kpreempt_disable();
2212 		my_cpu_id = CPU->cpu_id;
2213 		other_cpus = cpu_ready_set;
2214 		CPUSET_DEL(other_cpus, CPU->cpu_id);
2215 		target_found = 0;
2216 		if (!CPUSET_ISNULL(other_cpus)) {
2217 			/*
2218 			 * Pick the first one
2219 			 */
2220 			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2221 				if (cpu_id == my_cpu_id)
2222 					continue;
2223 
2224 				if (CPU_XCALL_READY(cpu_id)) {
2225 					target_cpu_id = cpu_id;
2226 					target_found = 1;
2227 					break;
2228 				}
2229 			}
2230 			ASSERT(target_found);
2231 
2232 			if (ptl1_panic_xc_one_test) {
2233 				xc_one(target_cpu_id,
2234 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2235 			}
2236 			if (ptl1_panic_xc_all_test) {
2237 				xc_some(other_cpus,
2238 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2239 			}
2240 			if (ptl1_panic_xt_one_test) {
2241 				xt_one(target_cpu_id,
2242 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2243 			}
2244 			if (ptl1_panic_xt_all_test) {
2245 				xt_some(other_cpus,
2246 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2247 			}
2248 		}
2249 		kpreempt_enable();
2250 		(void) timeout(ptl1_wakeup, NULL, hz);
2251 		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2252 	}
2253 	mutex_exit(&ptl1_mutex);
2254 }
2255 
2256 /*
2257  * Called during early startup to create the ptl1_thread
2258  */
2259 void
2260 init_ptl1_thread(void)
2261 {
2262 	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2263 	    &p0, TS_RUN, 0);
2264 }
2265 #endif	/* PTL1_PANIC_DEBUG */
2266 
2267 
2268 static void
2269 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2270 {
2271 	struct memlist *new;
2272 
2273 	new = *memlistp;
2274 	new->address = start;
2275 	new->size = len;
2276 	*memlistp = new + 1;
2277 }
2278 
2279 /*
2280  * Add to a memory list.
2281  * start = start of new memory segment
2282  * len = length of new memory segment in bytes
2283  * memlistp = pointer to array of available memory segment structures
2284  * curmemlistp = memory list to which to add segment.
2285  */
2286 static void
2287 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2288 	struct memlist **curmemlistp)
2289 {
2290 	struct memlist *new = *memlistp;
2291 
2292 	memlist_new(start, len, memlistp);
2293 	memlist_insert(new, curmemlistp);
2294 }
2295 
2296 static int
2297 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2298 {
2299 	int nseg;
2300 	size_t memseg_sz;
2301 	struct memseg *msp;
2302 
2303 	/*
2304 	 * The memseg list is for the chunks of physical memory that
2305 	 * will be managed by the vm system.  The number calculated is
2306 	 * a guess as boot may fragment it more when memory allocations
2307 	 * are made before kphysm_init().
2308 	 */
2309 	memseg_sz = (avail + 10) * sizeof (struct memseg);
2310 	memseg_sz = roundup(memseg_sz, PAGESIZE);
2311 	nseg = memseg_sz / sizeof (struct memseg);
2312 	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2313 	if (msp == NULL)
2314 		return (1);
2315 	PRM_DEBUG(memseg_free);
2316 
2317 	while (nseg--) {
2318 		msp->next = memseg_free;
2319 		memseg_free = msp;
2320 		msp++;
2321 	}
2322 	return (0);
2323 }
2324 
2325 /*
2326  * In the case of architectures that support dynamic addition of
2327  * memory at run-time there are two cases where memsegs need to
2328  * be initialized and added to the memseg list.
2329  * 1) memsegs that are constructed at startup.
2330  * 2) memsegs that are constructed at run-time on
2331  *    hot-plug capable architectures.
2332  * This code was originally part of the function kphysm_init().
2333  */
2334 
2335 static void
2336 memseg_list_add(struct memseg *memsegp)
2337 {
2338 	struct memseg **prev_memsegp;
2339 	pgcnt_t num;
2340 
2341 	/* insert in memseg list, decreasing number of pages order */
2342 
2343 	num = MSEG_NPAGES(memsegp);
2344 
2345 	for (prev_memsegp = &memsegs; *prev_memsegp;
2346 	    prev_memsegp = &((*prev_memsegp)->next)) {
2347 		if (num > MSEG_NPAGES(*prev_memsegp))
2348 			break;
2349 	}
2350 
2351 	memsegp->next = *prev_memsegp;
2352 	*prev_memsegp = memsegp;
2353 
2354 	if (kpm_enable) {
2355 		memsegp->nextpa = (memsegp->next) ?
2356 		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2357 
2358 		if (prev_memsegp != &memsegs) {
2359 			struct memseg *msp;
2360 			msp = (struct memseg *)((caddr_t)prev_memsegp -
2361 			    offsetof(struct memseg, next));
2362 			msp->nextpa = va_to_pa(memsegp);
2363 		} else {
2364 			memsegspa = va_to_pa(memsegs);
2365 		}
2366 	}
2367 }
2368 
2369 /*
2370  * PSM add_physmem_cb(). US-II and newer processors have some
2371  * flavor of the prefetch capability implemented. We exploit
2372  * this capability for optimum performance.
2373  */
2374 #define	PREFETCH_BYTES	64
2375 
2376 void
2377 add_physmem_cb(page_t *pp, pfn_t pnum)
2378 {
2379 	extern void	 prefetch_page_w(void *);
2380 
2381 	pp->p_pagenum = pnum;
2382 
2383 	/*
2384 	 * Prefetch one more page_t into E$. To prevent future
2385 	 * mishaps with the sizeof(page_t) changing on us, we
2386 	 * catch this on debug kernels if we can't bring in the
2387 	 * entire hpage with 2 PREFETCH_BYTES reads. See
2388 	 * also, sun4u/cpu/cpu_module.c
2389 	 */
2390 	/*LINTED*/
2391 	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2392 	prefetch_page_w((char *)pp);
2393 }
2394 
2395 /*
2396  * Find memseg with given pfn
2397  */
2398 static struct memseg *
2399 memseg_find(pfn_t base, pfn_t *next)
2400 {
2401 	struct memseg *seg;
2402 
2403 	if (next != NULL)
2404 		*next = LONG_MAX;
2405 	for (seg = memsegs; seg != NULL; seg = seg->next) {
2406 		if (base >= seg->pages_base && base < seg->pages_end)
2407 			return (seg);
2408 		if (next != NULL && seg->pages_base > base &&
2409 		    seg->pages_base < *next)
2410 			*next = seg->pages_base;
2411 	}
2412 	return (NULL);
2413 }
2414 
2415 extern struct vnode prom_ppages;
2416 
2417 /*
2418  * Put page allocated by OBP on prom_ppages
2419  */
2420 static void
2421 kphysm_erase(uint64_t addr, uint64_t len)
2422 {
2423 	struct page *pp;
2424 	struct memseg *seg;
2425 	pfn_t base = btop(addr), next;
2426 	pgcnt_t num = btop(len);
2427 
2428 	while (num != 0) {
2429 		pgcnt_t off, left;
2430 
2431 		seg = memseg_find(base, &next);
2432 		if (seg == NULL) {
2433 			if (next == LONG_MAX)
2434 				break;
2435 			left = MIN(next - base, num);
2436 			base += left, num -= left;
2437 			continue;
2438 		}
2439 		off = base - seg->pages_base;
2440 		pp = seg->pages + off;
2441 		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2442 		while (num != left) {
2443 			/*
2444 			 * init it, lock it, and hashin on prom_pages vp.
2445 			 *
2446 			 * XXX	vnode offsets on the prom_ppages vnode
2447 			 *	are page numbers (gack) for >32 bit
2448 			 *	physical memory machines.
2449 			 */
2450 			add_physmem_cb(pp, base);
2451 			if (page_trylock(pp, SE_EXCL) == 0)
2452 				cmn_err(CE_PANIC, "prom page locked");
2453 			(void) page_hashin(pp, &prom_ppages,
2454 			    (offset_t)base, NULL);
2455 			(void) page_pp_lock(pp, 0, 1);
2456 			pp++, base++, num--;
2457 		}
2458 	}
2459 }
2460 
2461 static page_t *ppnext;
2462 static pgcnt_t ppleft;
2463 
2464 static void *kpm_ppnext;
2465 static pgcnt_t kpm_ppleft;
2466 
2467 /*
2468  * Create a memseg
2469  */
2470 static void
2471 kphysm_memseg(uint64_t addr, uint64_t len)
2472 {
2473 	pfn_t base = btop(addr);
2474 	pgcnt_t num = btop(len);
2475 	struct memseg *seg;
2476 
2477 	seg = memseg_free;
2478 	memseg_free = seg->next;
2479 	ASSERT(seg != NULL);
2480 
2481 	seg->pages = ppnext;
2482 	seg->epages = ppnext + num;
2483 	seg->pages_base = base;
2484 	seg->pages_end = base + num;
2485 	ppnext += num;
2486 	ppleft -= num;
2487 
2488 	if (kpm_enable) {
2489 		pgcnt_t kpnum = ptokpmpr(num);
2490 
2491 		if (kpnum > kpm_ppleft)
2492 			panic("kphysm_memseg: kpm_pp overflow");
2493 		seg->pagespa = va_to_pa(seg->pages);
2494 		seg->epagespa = va_to_pa(seg->epages);
2495 		seg->kpm_pbase = kpmptop(ptokpmp(base));
2496 		seg->kpm_nkpmpgs = kpnum;
2497 		/*
2498 		 * In the kpm_smallpage case, the kpm array
2499 		 * is 1-1 wrt the page array
2500 		 */
2501 		if (kpm_smallpages) {
2502 			kpm_spage_t *kpm_pp = kpm_ppnext;
2503 
2504 			kpm_ppnext = kpm_pp + kpnum;
2505 			seg->kpm_spages = kpm_pp;
2506 			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2507 		} else {
2508 			kpm_page_t *kpm_pp = kpm_ppnext;
2509 
2510 			kpm_ppnext = kpm_pp + kpnum;
2511 			seg->kpm_pages = kpm_pp;
2512 			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2513 			/* ASSERT no kpm overlaps */
2514 			ASSERT(
2515 			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2516 			ASSERT(memseg_find(
2517 			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2518 		}
2519 		kpm_ppleft -= num;
2520 	}
2521 
2522 	memseg_list_add(seg);
2523 }
2524 
2525 /*
2526  * Add range to free list
2527  */
2528 void
2529 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2530 {
2531 	struct page *pp;
2532 	struct memseg *seg;
2533 	pfn_t base = btop(addr);
2534 	pgcnt_t num = btop(len);
2535 
2536 	seg = memseg_find(base, NULL);
2537 	ASSERT(seg != NULL);
2538 	pp = seg->pages + (base - seg->pages_base);
2539 
2540 	if (reclaim) {
2541 		struct page *rpp = pp;
2542 		struct page *lpp = pp + num;
2543 
2544 		/*
2545 		 * page should be locked on prom_ppages
2546 		 * unhash and unlock it
2547 		 */
2548 		while (rpp < lpp) {
2549 			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages);
2550 			page_pp_unlock(rpp, 0, 1);
2551 			page_hashout(rpp, NULL);
2552 			page_unlock(rpp);
2553 			rpp++;
2554 		}
2555 	}
2556 
2557 	/*
2558 	 * add_physmem() initializes the PSM part of the page
2559 	 * struct by calling the PSM back with add_physmem_cb().
2560 	 * In addition it coalesces pages into larger pages as
2561 	 * it initializes them.
2562 	 */
2563 	add_physmem(pp, num, base);
2564 }
2565 
2566 /*
2567  * kphysm_init() tackles the problem of initializing physical memory.
2568  */
2569 static void
2570 kphysm_init(void)
2571 {
2572 	struct memlist *pmem;
2573 
2574 	ASSERT(page_hash != NULL && page_hashsz != 0);
2575 
2576 	ppnext = pp_base;
2577 	ppleft = npages;
2578 	kpm_ppnext = kpm_pp_base;
2579 	kpm_ppleft = kpm_npages;
2580 
2581 	/*
2582 	 * installed pages not on nopp_memlist go in memseg list
2583 	 */
2584 	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2585 
2586 	/*
2587 	 * Free the avail list
2588 	 */
2589 	for (pmem = phys_avail; pmem != NULL; pmem = pmem->next)
2590 		kphysm_add(pmem->address, pmem->size, 0);
2591 
2592 	/*
2593 	 * Erase pages that aren't available
2594 	 */
2595 	diff_memlists(phys_install, phys_avail, kphysm_erase);
2596 
2597 	build_pfn_hash();
2598 }
2599 
2600 /*
2601  * Kernel VM initialization.
2602  * Assumptions about kernel address space ordering:
2603  *	(1) gap (user space)
2604  *	(2) kernel text
2605  *	(3) kernel data/bss
2606  *	(4) gap
2607  *	(5) kernel data structures
2608  *	(6) gap
2609  *	(7) debugger (optional)
2610  *	(8) monitor
2611  *	(9) gap (possibly null)
2612  *	(10) dvma
2613  *	(11) devices
2614  */
2615 static void
2616 kvm_init(void)
2617 {
2618 	/*
2619 	 * Put the kernel segments in kernel address space.
2620 	 */
2621 	rw_enter(&kas.a_lock, RW_WRITER);
2622 	as_avlinit(&kas);
2623 
2624 	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2625 	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2626 	(void) segkmem_create(&ktextseg);
2627 
2628 	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2629 	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2630 	(void) segkmem_create(&ktexthole);
2631 
2632 	(void) seg_attach(&kas, (caddr_t)valloc_base,
2633 	    (size_t)(econtig32 - valloc_base), &kvalloc);
2634 	(void) segkmem_create(&kvalloc);
2635 
2636 	if (kmem64_base) {
2637 		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2638 		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2639 		(void) segkmem_create(&kmem64);
2640 	}
2641 
2642 	/*
2643 	 * We're about to map out /boot.  This is the beginning of the
2644 	 * system resource management transition. We can no longer
2645 	 * call into /boot for I/O or memory allocations.
2646 	 */
2647 	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2648 	(void) segkmem_create(&kvseg);
2649 	hblk_alloc_dynamic = 1;
2650 
2651 	/*
2652 	 * we need to preallocate pages for DR operations before enabling large
2653 	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2654 	 */
2655 	memseg_remap_init();
2656 
2657 	/* at this point we are ready to use large page heap */
2658 	segkmem_heap_lp_init();
2659 
2660 	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2661 	    &kvseg32);
2662 	(void) segkmem_create(&kvseg32);
2663 
2664 	/*
2665 	 * Create a segment for the debugger.
2666 	 */
2667 	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2668 	(void) segkmem_create(&kdebugseg);
2669 
2670 	rw_exit(&kas.a_lock);
2671 }
2672 
2673 char obp_tte_str[] =
2674 	"h# %x constant MMU_PAGESHIFT "
2675 	"h# %x constant TTE8K "
2676 	"h# %x constant SFHME_SIZE "
2677 	"h# %x constant SFHME_TTE "
2678 	"h# %x constant HMEBLK_TAG "
2679 	"h# %x constant HMEBLK_NEXT "
2680 	"h# %x constant HMEBLK_MISC "
2681 	"h# %x constant HMEBLK_HME1 "
2682 	"h# %x constant NHMENTS "
2683 	"h# %x constant HBLK_SZMASK "
2684 	"h# %x constant HBLK_RANGE_SHIFT "
2685 	"h# %x constant HMEBP_HBLK "
2686 	"h# %x constant HMEBUCKET_SIZE "
2687 	"h# %x constant HTAG_SFMMUPSZ "
2688 	"h# %x constant HTAG_BSPAGE_SHIFT "
2689 	"h# %x constant HTAG_REHASH_SHIFT "
2690 	"h# %x constant SFMMU_INVALID_SHMERID "
2691 	"h# %x constant mmu_hashcnt "
2692 	"h# %p constant uhme_hash "
2693 	"h# %p constant khme_hash "
2694 	"h# %x constant UHMEHASH_SZ "
2695 	"h# %x constant KHMEHASH_SZ "
2696 	"h# %p constant KCONTEXT "
2697 	"h# %p constant KHATID "
2698 	"h# %x constant ASI_MEM "
2699 
2700 	": PHYS-X@ ( phys -- data ) "
2701 	"   ASI_MEM spacex@ "
2702 	"; "
2703 
2704 	": PHYS-W@ ( phys -- data ) "
2705 	"   ASI_MEM spacew@ "
2706 	"; "
2707 
2708 	": PHYS-L@ ( phys -- data ) "
2709 	"   ASI_MEM spaceL@ "
2710 	"; "
2711 
2712 	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2713 	"   3 * MMU_PAGESHIFT + "
2714 	"; "
2715 
2716 	": TTE_IS_VALID ( ttep -- flag ) "
2717 	"   PHYS-X@ 0< "
2718 	"; "
2719 
2720 	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2721 	"   dup TTE8K =  if "
2722 	"      drop HBLK_RANGE_SHIFT "
2723 	"   else "
2724 	"      TTE_PAGE_SHIFT "
2725 	"   then "
2726 	"; "
2727 
2728 	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2729 	"   tuck >> swap MMU_PAGESHIFT - << "
2730 	"; "
2731 
2732 	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2733 	"   >> over xor swap                    ( hash sfmmup ) "
2734 	"   KHATID <>  if                       ( hash ) "
2735 	"      UHMEHASH_SZ and                  ( bucket ) "
2736 	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2737 	"   else                                ( hash ) "
2738 	"      KHMEHASH_SZ and                  ( bucket ) "
2739 	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2740 	"   then                                ( hmebp ) "
2741 	"; "
2742 
2743 	": HME_HASH_TABLE_SEARCH "
2744 	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2745 	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2746 	"      dup if   		( sfmmup hmeblkp ) ( r: hblktag ) "
2747 	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2748 	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2749 	"		  true 	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2750 	"	     else						  "
2751 	"	     	  hmeblk_next + phys-x@ false 			  "
2752 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2753 	"	     then  						  "
2754 	"	  else							  "
2755 	"	     hmeblk_next + phys-x@ false 			  "
2756 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2757 	"	  then 							  "
2758 	"      else							  "
2759 	"         true 							  "
2760 	"      then  							  "
2761 	"   until r> drop 						  "
2762 	"; "
2763 
2764 	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2765 	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2766 	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2767 	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2768 	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2769 	"; "
2770 
2771 	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2772 	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2773 	"   TTE8K =  if                            ( hmeblkp addr ) "
2774 	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2775 	"   else                                   ( hmeblkp addr ) "
2776 	"      drop 0                              ( hmeblkp 0 ) "
2777 	"   then                                   ( hmeblkp hme-index ) "
2778 	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2779 	"   SFHME_TTE +                            ( ttep ) "
2780 	"; "
2781 
2782 	": unix-tte ( addr cnum -- false | tte-data true ) "
2783 	"    KCONTEXT = if                   ( addr ) "
2784 	"	KHATID                       ( addr khatid ) "
2785 	"    else                            ( addr ) "
2786 	"       drop false exit              ( false ) "
2787 	"    then "
2788 	"      ( addr khatid ) "
2789 	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2790 	"         2dup swap i HME_HASH_SHIFT  "
2791 					"( addr sfmmup sfmmup addr hmeshift ) "
2792 	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2793 	"         over i 4 pick               "
2794 				"( addr sfmmup hmebp sfmmup rehash addr ) "
2795 	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2796 	"         HME_HASH_TABLE_SEARCH       "
2797 					"( addr sfmmup { null | hmeblkp } ) "
2798 	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2799 	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2800 	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2801 	"               PHYS-X@ true          ( tte-data true ) "
2802 	"            else                     ( invalid-tte ) "
2803 	"               drop false            ( false ) "
2804 	"            then                     ( false | tte-data true ) "
2805 	"            unloop exit              ( false | tte-data true ) "
2806 	"         then                        ( addr sfmmup ) "
2807 	"      loop                           ( addr sfmmup ) "
2808 	"      2drop false                    ( false ) "
2809 	"; "
2810 ;
2811 
2812 void
2813 create_va_to_tte(void)
2814 {
2815 	char *bp;
2816 	extern int khmehash_num, uhmehash_num;
2817 	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2818 
2819 #define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2820 
2821 	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2822 
2823 	/*
2824 	 * Teach obp how to parse our sw ttes.
2825 	 */
2826 	(void) sprintf(bp, obp_tte_str,
2827 	    MMU_PAGESHIFT,
2828 	    TTE8K,
2829 	    sizeof (struct sf_hment),
2830 	    OFFSET(struct sf_hment, hme_tte),
2831 	    OFFSET(struct hme_blk, hblk_tag),
2832 	    OFFSET(struct hme_blk, hblk_nextpa),
2833 	    OFFSET(struct hme_blk, hblk_misc),
2834 	    OFFSET(struct hme_blk, hblk_hme),
2835 	    NHMENTS,
2836 	    HBLK_SZMASK,
2837 	    HBLK_RANGE_SHIFT,
2838 	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2839 	    sizeof (struct hmehash_bucket),
2840 	    HTAG_SFMMUPSZ,
2841 	    HTAG_BSPAGE_SHIFT,
2842 	    HTAG_REHASH_SHIFT,
2843 	    SFMMU_INVALID_SHMERID,
2844 	    mmu_hashcnt,
2845 	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
2846 	    (caddr_t)va_to_pa((caddr_t)khme_hash),
2847 	    UHMEHASH_SZ,
2848 	    KHMEHASH_SZ,
2849 	    KCONTEXT,
2850 	    KHATID,
2851 	    ASI_MEM);
2852 	prom_interpret(bp, 0, 0, 0, 0, 0);
2853 
2854 	kobj_free(bp, MMU_PAGESIZE);
2855 }
2856 
2857 void
2858 install_va_to_tte(void)
2859 {
2860 	/*
2861 	 * advise prom that he can use unix-tte
2862 	 */
2863 	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
2864 }
2865 
2866 /*
2867  * Here we add "device-type=console" for /os-io node, for currently
2868  * our kernel console output only supports displaying text and
2869  * performing cursor-positioning operations (through kernel framebuffer
2870  * driver) and it doesn't support other functionalities required for a
2871  * standard "display" device as specified in 1275 spec. The main missing
2872  * interface defined by the 1275 spec is "draw-logo".
2873  * also see the comments above prom_stdout_is_framebuffer().
2874  */
2875 static char *create_node =
2876 	"\" /\" find-device "
2877 	"new-device "
2878 	"\" os-io\" device-name "
2879 	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
2880 	": cb-r/w  ( adr,len method$ -- #read/#written ) "
2881 	"   2>r swap 2 2r> ['] $callback  catch  if "
2882 	"      2drop 3drop 0 "
2883 	"   then "
2884 	"; "
2885 	": read ( adr,len -- #read ) "
2886 	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
2887 	"       ( retN ... ret1 N ) "
2888 	"       ?dup  if "
2889 	"               swap >r 1-  0  ?do  drop  loop  r> "
2890 	"       else "
2891 	"               -2 "
2892 	"       then "
2893 	";    "
2894 	": write ( adr,len -- #written ) "
2895 	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
2896 	"       ( retN ... ret1 N ) "
2897 	"       ?dup  if "
2898 	"               swap >r 1-  0  ?do  drop  loop  r> "
2899 	"        else "
2900 	"               0 "
2901 	"       then "
2902 	"; "
2903 	": poll-tty ( -- ) ; "
2904 	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
2905 	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
2906 	": cb-give/take ( $method -- ) "
2907 	"       0 -rot ['] $callback catch  ?dup  if "
2908 	"               >r 2drop 2drop r> throw "
2909 	"       else "
2910 	"               0  ?do  drop  loop "
2911 	"       then "
2912 	"; "
2913 	": give ( -- )  \" exit-input\" cb-give/take ; "
2914 	": take ( -- )  \" enter-input\" cb-give/take ; "
2915 	": open ( -- ok? )  true ; "
2916 	": close ( -- ) ; "
2917 	"finish-device "
2918 	"device-end ";
2919 
2920 /*
2921  * Create the OBP input/output node (FCode serial driver).
2922  * It is needed for both USB console keyboard and for
2923  * the kernel terminal emulator.  It is too early to check for a
2924  * kernel console compatible framebuffer now, so we create this
2925  * so that we're ready if we need to enable kernel terminal emulation.
2926  *
2927  * When the USB software takes over the input device at the time
2928  * consconfig runs, OBP's stdin is redirected to this node.
2929  * Whenever the FORTH user interface is used after this switch,
2930  * the node will call back into the kernel for console input.
2931  * If a serial device such as ttya or a UART with a Type 5 keyboard
2932  * attached is used, OBP takes over the serial device when the system
2933  * goes to the debugger after the system is booted.  This sharing
2934  * of the relatively simple serial device is difficult but possible.
2935  * Sharing the USB host controller is impossible due its complexity.
2936  *
2937  * Similarly to USB keyboard input redirection, after consconfig_dacf
2938  * configures a kernel console framebuffer as the standard output
2939  * device, OBP's stdout is switched to to vector through the
2940  * /os-io node into the kernel terminal emulator.
2941  */
2942 static void
2943 startup_create_io_node(void)
2944 {
2945 	prom_interpret(create_node, 0, 0, 0, 0, 0);
2946 }
2947 
2948 
2949 static void
2950 do_prom_version_check(void)
2951 {
2952 	int i;
2953 	pnode_t node;
2954 	char buf[64];
2955 	static char drev[] = "Down-rev firmware detected%s\n"
2956 	    "\tPlease upgrade to the following minimum version:\n"
2957 	    "\t\t%s\n";
2958 
2959 	i = prom_version_check(buf, sizeof (buf), &node);
2960 
2961 	if (i == PROM_VER64_OK)
2962 		return;
2963 
2964 	if (i == PROM_VER64_UPGRADE) {
2965 		cmn_err(CE_WARN, drev, "", buf);
2966 
2967 #ifdef	DEBUG
2968 		prom_enter_mon();	/* Type 'go' to continue */
2969 		cmn_err(CE_WARN, "Booting with down-rev firmware\n");
2970 		return;
2971 #else
2972 		halt(0);
2973 #endif
2974 	}
2975 
2976 	/*
2977 	 * The other possibility is that this is a server running
2978 	 * good firmware, but down-rev firmware was detected on at
2979 	 * least one other cpu board. We just complain if we see
2980 	 * that.
2981 	 */
2982 	cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
2983 }
2984 
2985 
2986 /*
2987  * Must be defined in platform dependent code.
2988  */
2989 extern caddr_t modtext;
2990 extern size_t modtext_sz;
2991 extern caddr_t moddata;
2992 
2993 #define	HEAPTEXT_ARENA(addr)	\
2994 	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
2995 	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
2996 	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
2997 
2998 #define	HEAPTEXT_OVERSIZED(addr)	\
2999 	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3000 
3001 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3002 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3003 kmutex_t texthole_lock;
3004 
3005 char kern_bootargs[OBP_MAXPATHLEN];
3006 
3007 void
3008 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3009 {
3010 	uintptr_t addr, limit;
3011 
3012 	addr = HEAPTEXT_BASE;
3013 	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3014 
3015 	/*
3016 	 * Before we initialize the text_arena, we want to punch holes in the
3017 	 * underlying heaptext_arena.  This guarantees that for any text
3018 	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3019 	 */
3020 	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3021 	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3022 		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3023 		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3024 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3025 	}
3026 
3027 	/*
3028 	 * Allocate one page at the oversize to break up the text region
3029 	 * from the oversized region.
3030 	 */
3031 	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3032 	    (void *)limit, (void *)(limit + PAGESIZE),
3033 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3034 
3035 	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3036 	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3037 	    heaptext_arena, 0, VM_SLEEP);
3038 	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3039 	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3040 }
3041 
3042 caddr_t
3043 kobj_text_alloc(vmem_t *arena, size_t size)
3044 {
3045 	caddr_t rval, better;
3046 
3047 	/*
3048 	 * First, try a sleeping allocation.
3049 	 */
3050 	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3051 
3052 	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3053 		return (rval);
3054 
3055 	/*
3056 	 * We didn't get the area that we wanted.  We're going to try to do an
3057 	 * allocation with explicit constraints.
3058 	 */
3059 	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3060 	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3061 	    VM_NOSLEEP | VM_BESTFIT);
3062 
3063 	if (better != NULL) {
3064 		/*
3065 		 * That worked.  Free our first attempt and return.
3066 		 */
3067 		vmem_free(arena, rval, size);
3068 		return (better);
3069 	}
3070 
3071 	/*
3072 	 * That didn't work; we'll have to return our first attempt.
3073 	 */
3074 	return (rval);
3075 }
3076 
3077 caddr_t
3078 kobj_texthole_alloc(caddr_t addr, size_t size)
3079 {
3080 	int arena = HEAPTEXT_ARENA(addr);
3081 	char c[30];
3082 	uintptr_t base;
3083 
3084 	if (HEAPTEXT_OVERSIZED(addr)) {
3085 		/*
3086 		 * If this is an oversized allocation, there is no text hole
3087 		 * available for it; return NULL.
3088 		 */
3089 		return (NULL);
3090 	}
3091 
3092 	mutex_enter(&texthole_lock);
3093 
3094 	if (texthole_arena[arena] == NULL) {
3095 		ASSERT(texthole_source[arena] == NULL);
3096 
3097 		if (arena == 0) {
3098 			texthole_source[0] = vmem_create("module_text_holesrc",
3099 			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3100 			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3101 			    0, VM_SLEEP);
3102 		} else {
3103 			base = HEAPTEXT_BASE +
3104 			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3105 
3106 			(void) snprintf(c, sizeof (c),
3107 			    "heaptext_holesrc_%d", arena);
3108 
3109 			texthole_source[arena] = vmem_create(c, (void *)base,
3110 			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3111 			    0, VM_SLEEP);
3112 		}
3113 
3114 		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3115 
3116 		texthole_arena[arena] = vmem_create(c, NULL, 0,
3117 		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3118 		    texthole_source[arena], 0, VM_SLEEP);
3119 	}
3120 
3121 	mutex_exit(&texthole_lock);
3122 
3123 	ASSERT(texthole_arena[arena] != NULL);
3124 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3125 	return (vmem_alloc(texthole_arena[arena], size,
3126 	    VM_BESTFIT | VM_NOSLEEP));
3127 }
3128 
3129 void
3130 kobj_texthole_free(caddr_t addr, size_t size)
3131 {
3132 	int arena = HEAPTEXT_ARENA(addr);
3133 
3134 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3135 	ASSERT(texthole_arena[arena] != NULL);
3136 	vmem_free(texthole_arena[arena], addr, size);
3137 }
3138 
3139 void
3140 release_bootstrap(void)
3141 {
3142 	if (&cif_init)
3143 		cif_init();
3144 }
3145