xref: /illumos-gate/usr/src/uts/sun4/os/startup.c (revision 9ccfd38e64be4d6a2c01daa15fe7f5627122bdc2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2016 by Delphix. All rights reserved.
25  * Copyright 2018 Joyent, Inc.
26  * Copyright 2019 Peter Tribble.
27  */
28 
29 #include <sys/machsystm.h>
30 #include <sys/archsystm.h>
31 #include <sys/vm.h>
32 #include <sys/cpu.h>
33 #include <sys/atomic.h>
34 #include <sys/reboot.h>
35 #include <sys/kdi.h>
36 #include <sys/bootconf.h>
37 #include <sys/memlist_plat.h>
38 #include <sys/memlist_impl.h>
39 #include <sys/prom_plat.h>
40 #include <sys/prom_isa.h>
41 #include <sys/autoconf.h>
42 #include <sys/ivintr.h>
43 #include <sys/fpu/fpusystm.h>
44 #include <sys/iommutsb.h>
45 #include <vm/vm_dep.h>
46 #include <vm/seg_dev.h>
47 #include <vm/seg_kmem.h>
48 #include <vm/seg_kpm.h>
49 #include <vm/seg_map.h>
50 #include <vm/seg_kp.h>
51 #include <sys/sysconf.h>
52 #include <vm/hat_sfmmu.h>
53 #include <sys/kobj.h>
54 #include <sys/sun4asi.h>
55 #include <sys/clconf.h>
56 #include <sys/platform_module.h>
57 #include <sys/panic.h>
58 #include <sys/cpu_sgnblk_defs.h>
59 #include <sys/clock.h>
60 #include <sys/cmn_err.h>
61 #include <sys/dumphdr.h>
62 #include <sys/promif.h>
63 #include <sys/prom_debug.h>
64 #include <sys/traptrace.h>
65 #include <sys/memnode.h>
66 #include <sys/mem_cage.h>
67 #include <sys/mmu.h>
68 #include <sys/swap.h>
69 
70 extern void setup_trap_table(void);
71 extern int cpu_intrq_setup(struct cpu *);
72 extern void cpu_intrq_register(struct cpu *);
73 extern void contig_mem_init(void);
74 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
75 extern void mach_dump_buffer_init(void);
76 extern void mach_descrip_init(void);
77 extern void mach_descrip_startup_fini(void);
78 extern void mach_memscrub(void);
79 extern void mach_fpras(void);
80 extern void mach_cpu_halt_idle(void);
81 extern void mach_hw_copy_limit(void);
82 extern void load_mach_drivers(void);
83 extern void load_tod_module(void);
84 #pragma weak load_tod_module
85 
86 extern int ndata_alloc_mmfsa(struct memlist *ndata);
87 #pragma weak ndata_alloc_mmfsa
88 
89 extern void cif_init(void);
90 #pragma weak cif_init
91 
92 extern void parse_idprom(void);
93 extern void add_vx_handler(char *, int, void (*)(cell_t *));
94 extern void mem_config_init(void);
95 extern void memseg_remap_init(void);
96 
97 extern void mach_kpm_init(void);
98 extern void pcf_init();
99 extern int size_pse_array(pgcnt_t, int);
100 extern void pg_init();
101 
102 /*
103  * External Data:
104  */
105 extern int vac_size;	/* cache size in bytes */
106 extern uint_t vac_mask;	/* VAC alignment consistency mask */
107 extern uint_t vac_colors;
108 
109 /*
110  * Global Data Definitions:
111  */
112 
113 /*
114  * XXX - Don't port this to new architectures
115  * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
116  * 'romp' has no use with a prom with an IEEE 1275 client interface.
117  * The driver doesn't use the value, but it depends on the symbol.
118  */
119 void *romp;		/* veritas driver won't load without romp 4154976 */
120 /*
121  * Declare these as initialized data so we can patch them.
122  */
123 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
124 pgcnt_t segkpsize =
125     btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
126 uint_t segmap_percent = 6; /* Size of segmap segment */
127 
128 int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
129 int vac_copyback = 1;
130 char *cache_mode = NULL;
131 int use_mix = 1;
132 int prom_debug = 0;
133 
134 caddr_t boot_tba;		/* %tba at boot - used by kmdb */
135 uint_t	tba_taken_over = 0;
136 
137 caddr_t s_text;			/* start of kernel text segment */
138 caddr_t e_text;			/* end of kernel text segment */
139 caddr_t s_data;			/* start of kernel data segment */
140 caddr_t e_data;			/* end of kernel data segment */
141 
142 caddr_t modtext;		/* beginning of module text */
143 size_t	modtext_sz;		/* size of module text */
144 caddr_t moddata;		/* beginning of module data reserve */
145 caddr_t e_moddata;		/* end of module data reserve */
146 
147 /*
148  * End of first block of contiguous kernel in 32-bit virtual address space
149  */
150 caddr_t		econtig32;	/* end of first blk of contiguous kernel */
151 
152 caddr_t		ncbase;		/* beginning of non-cached segment */
153 caddr_t		ncend;		/* end of non-cached segment */
154 
155 size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
156 caddr_t	nalloc_base;		/* beginning of nucleus allocation */
157 caddr_t nalloc_end;		/* end of nucleus allocatable memory */
158 caddr_t valloc_base;		/* beginning of kvalloc segment	*/
159 
160 caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
161 caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
162 size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
163 caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
164 int	kmem64_szc;		/* page size code */
165 uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
166 
167 uintptr_t shm_alignment;	/* VAC address consistency modulus */
168 struct memlist *phys_install;	/* Total installed physical memory */
169 struct memlist *phys_avail;	/* Available (unreserved) physical memory */
170 struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
171 struct memlist *nopp_list;	/* pages with no backing page structs */
172 struct memlist ndata;		/* memlist of nucleus allocatable memory */
173 int memexp_flag;		/* memory expansion card flag */
174 uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
175 pgcnt_t obp_pages;		/* Physical pages used by OBP */
176 
177 /*
178  * VM data structures
179  */
180 long page_hashsz;		/* Size of page hash table (power of two) */
181 unsigned int page_hashsz_shift;	/* log2(page_hashsz) */
182 struct page *pp_base;		/* Base of system page struct array */
183 size_t pp_sz;			/* Size in bytes of page struct array */
184 struct page **page_hash;	/* Page hash table */
185 pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
186 size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
187 int pse_shift;			/* log2(pse_table_size) */
188 struct seg ktextseg;		/* Segment used for kernel executable image */
189 struct seg kvalloc;		/* Segment used for "valloc" mapping */
190 struct seg kpseg;		/* Segment used for pageable kernel virt mem */
191 struct seg ktexthole;		/* Segment used for nucleus text hole */
192 struct seg kmapseg;		/* Segment used for generic kernel mappings */
193 struct seg kpmseg;		/* Segment used for physical mapping */
194 struct seg kdebugseg;		/* Segment used for the kernel debugger */
195 
196 void *kpm_pp_base;		/* Base of system kpm_page array */
197 size_t	kpm_pp_sz;		/* Size of system kpm_page array */
198 pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
199 
200 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
201 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
202 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
203 
204 int segzio_fromheap = 0;	/* zio allocations occur from heap */
205 caddr_t segzio_base;		/* Base address of segzio */
206 pgcnt_t segziosize = 0;		/* size of zio segment in pages */
207 
208 /*
209  * A static DR page_t VA map is reserved that can map the page structures
210  * for a domain's entire RA space. The pages that backs this space are
211  * dynamically allocated and need not be physically contiguous.  The DR
212  * map size is derived from KPM size.
213  */
214 int ppvm_enable = 0;		/* Static virtual map for page structs */
215 page_t *ppvm_base;		/* Base of page struct map */
216 pgcnt_t ppvm_size = 0;		/* Size of page struct map */
217 
218 /*
219  * debugger pages (if allocated)
220  */
221 struct vnode kdebugvp;
222 
223 /*
224  * VA range available to the debugger
225  */
226 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
227 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
228 
229 /*
230  * Segment for relocated kernel structures in 64-bit large RAM kernels
231  */
232 struct seg kmem64;
233 
234 struct memseg *memseg_free;
235 
236 struct vnode unused_pages_vp;
237 
238 /*
239  * VM data structures allocated early during boot.
240  */
241 size_t pagehash_sz;
242 uint64_t memlist_sz;
243 
244 char tbr_wr_addr_inited = 0;
245 
246 caddr_t	mpo_heap32_buf = NULL;
247 size_t	mpo_heap32_bufsz = 0;
248 
249 /*
250  * Static Routines:
251  */
252 static int ndata_alloc_memseg(struct memlist *, size_t);
253 static void memlist_new(uint64_t, uint64_t, struct memlist **);
254 static void memlist_add(uint64_t, uint64_t,
255 	struct memlist **, struct memlist **);
256 static void kphysm_init(void);
257 static void kvm_init(void);
258 static void install_kmem64_tte(void);
259 
260 static void startup_init(void);
261 static void startup_memlist(void);
262 static void startup_modules(void);
263 static void startup_bop_gone(void);
264 static void startup_vm(void);
265 static void startup_end(void);
266 static void setup_cage_params(void);
267 static void startup_create_io_node(void);
268 
269 static pgcnt_t npages;
270 static struct memlist *memlist;
271 void *memlist_end;
272 
273 static pgcnt_t bop_alloc_pages;
274 static caddr_t hblk_base;
275 uint_t hblk_alloc_dynamic = 0;
276 uint_t hblk1_min = H1MIN;
277 
278 
279 /*
280  * After receiving a thermal interrupt, this is the number of seconds
281  * to delay before shutting off the system, assuming
282  * shutdown fails.  Use /etc/system to change the delay if this isn't
283  * large enough.
284  */
285 int thermal_powerdown_delay = 1200;
286 
287 /*
288  * Used to hold off page relocations into the cage until OBP has completed
289  * its boot-time handoff of its resources to the kernel.
290  */
291 int page_relocate_ready = 0;
292 
293 /*
294  * Indicate if kmem64 allocation was done in small chunks
295  */
296 int kmem64_smchunks = 0;
297 
298 /*
299  * Enable some debugging messages concerning memory usage...
300  */
301 #ifdef  DEBUGGING_MEM
302 static int debugging_mem;
303 static void
304 printmemlist(char *title, struct memlist *list)
305 {
306 	if (!debugging_mem)
307 		return;
308 
309 	printf("%s\n", title);
310 
311 	while (list) {
312 		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
313 		    (uint32_t)(list->ml_address >> 32),
314 		    (uint32_t)list->ml_address,
315 		    (uint32_t)(list->ml_size >> 32),
316 		    (uint32_t)(list->ml_size));
317 		list = list->ml_next;
318 	}
319 }
320 
321 void
322 printmemseg(struct memseg *memseg)
323 {
324 	if (!debugging_mem)
325 		return;
326 
327 	printf("memseg\n");
328 
329 	while (memseg) {
330 		prom_printf("\tpage = 0x%p, epage = 0x%p, "
331 		    "pfn = 0x%x, epfn = 0x%x\n",
332 		    memseg->pages, memseg->epages,
333 		    memseg->pages_base, memseg->pages_end);
334 		memseg = memseg->next;
335 	}
336 }
337 
338 #define	debug_pause(str)	halt((str))
339 #define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
340 #define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
341 #define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
342 #define	MPRINTF3(str, a, b, c) \
343 	if (debugging_mem) prom_printf((str), (a), (b), (c))
344 #else	/* DEBUGGING_MEM */
345 #define	MPRINTF(str)
346 #define	MPRINTF1(str, a)
347 #define	MPRINTF2(str, a, b)
348 #define	MPRINTF3(str, a, b, c)
349 #endif	/* DEBUGGING_MEM */
350 
351 
352 /*
353  *
354  *                    Kernel's Virtual Memory Layout.
355  *                       /-----------------------\
356  * 0xFFFFFFFF.FFFFFFFF  -|                       |-
357  *                       |   OBP's virtual page  |
358  *                       |        tables         |
359  * 0xFFFFFFFC.00000000  -|-----------------------|-
360  *                       :                       :
361  *                       :                       :
362  *                      -|-----------------------|-
363  *                       |       segzio          | (base and size vary)
364  * 0xFFFFFE00.00000000  -|-----------------------|-
365  *                       |                       |  Ultrasparc I/II support
366  *                       |    segkpm segment     |  up to 2TB of physical
367  *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
368  *                       |                       |
369  * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
370  *                       :                       :
371  *                       :                       :
372  * 0xFFFFF810.00000000  -|-----------------------|- hole_end
373  *                       |                       |      ^
374  *                       |  UltraSPARC I/II call |      |
375  *                       | bug requires an extra |      |
376  *                       | 4 GB of space between |      |
377  *                       |   hole and used RAM   |	|
378  *                       |                       |      |
379  * 0xFFFFF800.00000000  -|-----------------------|-     |
380  *                       |                       |      |
381  *                       | Virtual Address Hole  |   UltraSPARC
382  *                       |  on UltraSPARC I/II   |  I/II * ONLY *
383  *                       |                       |      |
384  * 0x00000800.00000000  -|-----------------------|-     |
385  *                       |                       |      |
386  *                       |  UltraSPARC I/II call |      |
387  *                       | bug requires an extra |      |
388  *                       | 4 GB of space between |      |
389  *                       |   hole and used RAM   |      |
390  *                       |                       |      v
391  * 0x000007FF.00000000  -|-----------------------|- hole_start -----
392  *                       :                       :		   ^
393  *                       :                       :		   |
394  *                       |-----------------------|                 |
395  *                       |                       |                 |
396  *                       |  ecache flush area    |                 |
397  *                       |  (twice largest e$)   |                 |
398  *                       |                       |                 |
399  * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
400  *                       | overmapped area       |   alignend_end  |
401  *                       | (kmem64_alignsize     |		   |
402  *                       |  boundary)            |		   |
403  * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
404  *                       |                       |		   |
405  *                       |   64-bit kernel ONLY  |		   |
406  *                       |                       |		   |
407  *                       |    kmem64 segment     |		   |
408  *                       |                       |		   |
409  *                       | (Relocated extra HME  |	     Approximately
410  *                       |   block allocations,  |	    1 TB of virtual
411  *                       |   memnode freelists,  |	     address space
412  *                       |    HME hash buckets,  |		   |
413  *                       | mml_table, kpmp_table,|		   |
414  *                       |  page_t array and     |		   |
415  *                       |  hashblock pool to    |		   |
416  *                       |   avoid hard-coded    |		   |
417  *                       |     32-bit vaddr      |		   |
418  *                       |     limitations)      |		   |
419  *                       |                       |		   v
420  * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
421  *                       |                       |
422  *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
423  *                       |                       |
424  * 0x00000300.00000000  -|-----------------------|- SYSBASE
425  *                       :                       :
426  *                       :                       :
427  *                      -|-----------------------|-
428  *                       |                       |
429  *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
430  *                       |                       |               256G MAX)
431  * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
432  *                       :                       :
433  *                       :                       :
434  *                      -|-----------------------|-
435  *                       |                       |
436  *                       |       segkp           |    SEGKPSIZE (2GB)
437  *                       |                       |
438  *                       |                       |
439  * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
440  *                       |                       |
441  * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
442  *                       |                       |       (SEGKPBASE - 0x400000)
443  * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
444  *                       |                       |       (MEMSCRUBBASE - NCARGS)
445  * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
446  *                       |                       |       (ARGSBASE - PPMAPSIZE)
447  * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
448  *                       |                       |
449  * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
450  *                       |                       |
451  * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
452  *                       :                       :
453  *                       :                       :
454  * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
455  *                       |                       |
456  *                       |         OBP           |
457  *                       |                       |
458  * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
459  *                       |         kmdb          |
460  * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
461  *                       :                       :
462  *                       :                       :
463  * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
464  *                       |                       |
465  *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
466  *                       |                       |    ~64MB)
467  *			-|-----------------------|
468  *			 |	IVSIZE		 |
469  * 0x00000000.70004000  -|-----------------------|
470  *                       |     panicbuf          |
471  * 0x00000000.70002000	-|-----------------------|
472  *			 |	PAGESIZE	 |
473  * 0x00000000.70000000  -|-----------------------|- SYSBASE32
474  *                       |       boot-time       |
475  *                       |    temporary space    |
476  * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
477  *                       :                       :
478  *                       :                       :
479  *                       |                       |
480  *                       |-----------------------|- econtig32
481  *                       |    vm structures      |
482  * 0x00000000.01C00000   |-----------------------|- nalloc_end
483  *                       |         TSBs          |
484  *                       |-----------------------|- end/nalloc_base
485  *                       |   kernel data & bss   |
486  * 0x00000000.01800000  -|-----------------------|
487  *                       :   nucleus text hole   :
488  * 0x00000000.01400000  -|-----------------------|
489  *                       :                       :
490  *                       |-----------------------|
491  *                       |      module text      |
492  *                       |-----------------------|- e_text/modtext
493  *                       |      kernel text      |
494  *                       |-----------------------|
495  *                       |    trap table (48k)   |
496  * 0x00000000.01000000  -|-----------------------|- KERNELBASE
497  *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
498  *                       |-----------------------|
499  *                       |                       |
500  *                       |        invalid        |
501  *                       |                       |
502  * 0x00000000.00000000  _|_______________________|
503  *
504  *
505  *
506  *                   32-bit User Virtual Memory Layout.
507  *                       /-----------------------\
508  *                       |                       |
509  *                       |        invalid        |
510  *                       |                       |
511  *          0xFFC00000  -|-----------------------|- USERLIMIT
512  *                       |       user stack      |
513  *                       :                       :
514  *                       :                       :
515  *                       :                       :
516  *                       |       user data       |
517  *                      -|-----------------------|-
518  *                       |       user text       |
519  *          0x00002000  -|-----------------------|-
520  *                       |       invalid         |
521  *          0x00000000  _|_______________________|
522  *
523  *
524  *
525  *                   64-bit User Virtual Memory Layout.
526  *                       /-----------------------\
527  *                       |                       |
528  *                       |        invalid        |
529  *                       |                       |
530  *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
531  *                       |       user stack      |
532  *                       :                       :
533  *                       :                       :
534  *                       :                       :
535  *                       |       user data       |
536  *                      -|-----------------------|-
537  *                       |       user text       |
538  *  0x00000000.01000000 -|-----------------------|-
539  *                       |       invalid         |
540  *  0x00000000.00000000 _|_______________________|
541  */
542 
543 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
544 extern uint64_t ecache_flush_address(void);
545 
546 #pragma weak load_platform_modules
547 #pragma weak plat_startup_memlist
548 #pragma weak ecache_init_scrub_flush_area
549 #pragma weak ecache_flush_address
550 
551 
552 /*
553  * By default the DR Cage is enabled for maximum OS
554  * MPSS performance.  Users needing to disable the cage mechanism
555  * can set this variable to zero via /etc/system.
556  * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
557  * will result in loss of DR functionality.
558  * Platforms wishing to disable kernel Cage by default
559  * should do so in their set_platform_defaults() routine.
560  */
561 int	kernel_cage_enable = 1;
562 
563 static void
564 setup_cage_params(void)
565 {
566 	void (*func)(void);
567 
568 	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
569 	if (func != NULL) {
570 		(*func)();
571 		return;
572 	}
573 
574 	if (kernel_cage_enable == 0) {
575 		return;
576 	}
577 	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
578 
579 	if (kcage_on) {
580 		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
581 	} else {
582 		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
583 	}
584 
585 }
586 
587 /*
588  * Machine-dependent startup code
589  */
590 void
591 startup(void)
592 {
593 	startup_init();
594 	if (&startup_platform)
595 		startup_platform();
596 	startup_memlist();
597 	startup_modules();
598 	setup_cage_params();
599 	startup_bop_gone();
600 	startup_vm();
601 	startup_end();
602 }
603 
604 struct regs sync_reg_buf;
605 uint64_t sync_tt;
606 
607 void
608 sync_handler(void)
609 {
610 	struct  panic_trap_info	ti;
611 	int i;
612 
613 	/*
614 	 * Prevent trying to talk to the other CPUs since they are
615 	 * sitting in the prom and won't reply.
616 	 */
617 	for (i = 0; i < NCPU; i++) {
618 		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
619 			cpu[i]->cpu_flags &= ~CPU_READY;
620 			cpu[i]->cpu_flags |= CPU_QUIESCED;
621 			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
622 		}
623 	}
624 
625 	/*
626 	 * Force a serial dump, since there are no CPUs to help.
627 	 */
628 	dump_plat_mincpu = 0;
629 
630 	/*
631 	 * We've managed to get here without going through the
632 	 * normal panic code path. Try and save some useful
633 	 * information.
634 	 */
635 	if (!panicstr && (curthread->t_panic_trap == NULL)) {
636 		ti.trap_type = sync_tt;
637 		ti.trap_regs = &sync_reg_buf;
638 		ti.trap_addr = NULL;
639 		ti.trap_mmu_fsr = 0x0;
640 
641 		curthread->t_panic_trap = &ti;
642 	}
643 
644 	/*
645 	 * If we're re-entering the panic path, update the signature
646 	 * block so that the SC knows we're in the second part of panic.
647 	 */
648 	if (panicstr)
649 		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
650 
651 	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
652 	panic("sync initiated");
653 }
654 
655 
656 static void
657 startup_init(void)
658 {
659 	/*
660 	 * We want to save the registers while we're still in OBP
661 	 * so that we know they haven't been fiddled with since.
662 	 * (In principle, OBP can't change them just because it
663 	 * makes a callback, but we'd rather not depend on that
664 	 * behavior.)
665 	 */
666 	char		sync_str[] =
667 	    "warning @ warning off : sync "
668 	    "%%tl-c %%tstate h# %p x! "
669 	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
670 	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
671 	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
672 	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
673 	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
674 	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
675 	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
676 	    "sync ; warning !";
677 
678 	/*
679 	 * 20 == num of %p substrings
680 	 * 16 == max num of chars %p will expand to.
681 	 */
682 	char		bp[sizeof (sync_str) + 16 * 20];
683 
684 	/*
685 	 * Initialize ptl1 stack for the 1st CPU.
686 	 */
687 	ptl1_init_cpu(&cpu0);
688 
689 	/*
690 	 * Initialize the address map for cache consistent mappings
691 	 * to random pages; must be done after vac_size is set.
692 	 */
693 	ppmapinit();
694 
695 	/*
696 	 * Initialize the PROM callback handler.
697 	 */
698 	init_vx_handler();
699 
700 	/*
701 	 * have prom call sync_callback() to handle the sync and
702 	 * save some useful information which will be stored in the
703 	 * core file later.
704 	 */
705 	(void) sprintf((char *)bp, sync_str,
706 	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
707 	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
708 	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
709 	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
710 	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
711 	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
712 	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
713 	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
714 	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
715 	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
716 	prom_interpret(bp, 0, 0, 0, 0, 0);
717 	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
718 }
719 
720 
721 size_t
722 calc_pp_sz(pgcnt_t npages)
723 {
724 
725 	return (npages * sizeof (struct page));
726 }
727 
728 size_t
729 calc_kpmpp_sz(pgcnt_t npages)
730 {
731 
732 	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
733 	kpm_pgsz = 1ull << kpm_pgshft;
734 	kpm_pgoff = kpm_pgsz - 1;
735 	kpmp2pshft = kpm_pgshft - PAGESHIFT;
736 	kpmpnpgs = 1 << kpmp2pshft;
737 
738 	if (kpm_smallpages == 0) {
739 		/*
740 		 * Avoid fragmentation problems in kphysm_init()
741 		 * by allocating for all of physical memory
742 		 */
743 		kpm_npages = ptokpmpr(physinstalled);
744 		return (kpm_npages * sizeof (kpm_page_t));
745 	} else {
746 		kpm_npages = npages;
747 		return (kpm_npages * sizeof (kpm_spage_t));
748 	}
749 }
750 
751 size_t
752 calc_pagehash_sz(pgcnt_t npages)
753 {
754 	/* LINTED */
755 	ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
756 	/*
757 	 * The page structure hash table size is a power of 2
758 	 * such that the average hash chain length is PAGE_HASHAVELEN.
759 	 */
760 	page_hashsz = npages / PAGE_HASHAVELEN;
761 	page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
762 	    highbit(page_hashsz));
763 	page_hashsz = 1 << page_hashsz_shift;
764 	return (page_hashsz * sizeof (struct page *));
765 }
766 
767 int testkmem64_smchunks = 0;
768 
769 int
770 alloc_kmem64(caddr_t base, caddr_t end)
771 {
772 	int i;
773 	caddr_t aligned_end = NULL;
774 
775 	if (testkmem64_smchunks)
776 		return (1);
777 
778 	/*
779 	 * Make one large memory alloc after figuring out the 64-bit size. This
780 	 * will enable use of the largest page size appropriate for the system
781 	 * architecture.
782 	 */
783 	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
784 	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
785 	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
786 		size_t alloc_size, alignsize;
787 #if !defined(C_OBP)
788 		unsigned long long pa;
789 #endif	/* !C_OBP */
790 
791 		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
792 			continue;
793 		alignsize = TTEBYTES(i);
794 		kmem64_szc = i;
795 
796 		/* limit page size for small memory */
797 		if (mmu_btop(alignsize) > (npages >> 2))
798 			continue;
799 
800 		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
801 		alloc_size = aligned_end - base;
802 #if !defined(C_OBP)
803 		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
804 			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
805 				kmem64_pabase = pa;
806 				kmem64_aligned_end = aligned_end;
807 				install_kmem64_tte();
808 				break;
809 			} else {
810 				prom_free_phys(alloc_size, pa);
811 			}
812 		}
813 #else	/* !C_OBP */
814 		if (prom_alloc(base, alloc_size, alignsize) == base) {
815 			kmem64_pabase = va_to_pa(kmem64_base);
816 			kmem64_aligned_end = aligned_end;
817 			break;
818 		}
819 #endif	/* !C_OBP */
820 		if (i == TTE8K) {
821 #ifdef sun4v
822 			/* return failure to try small allocations */
823 			return (1);
824 #else
825 			prom_panic("kmem64 allocation failure");
826 #endif
827 		}
828 	}
829 	ASSERT(aligned_end != NULL);
830 	return (0);
831 }
832 
833 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
834 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
835 
836 #if !defined(C_OBP)
837 /*
838  * Install a temporary tte handler in OBP for kmem64 area.
839  *
840  * We map kmem64 area with large pages before the trap table is taken
841  * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
842  * the same area. Duplicate tlb entries with different page sizes
843  * cause unpredicatble behavior.  To avoid this, we don't create
844  * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
845  * OBP).  Instead, we manage translations with a temporary va>tte-data
846  * handler (kmem64-tte).  This handler is replaced by unix-tte when
847  * the trap table is taken over.
848  *
849  * The temporary handler knows the physical address of the kmem64
850  * area. It uses the prom's pgmap@ Forth word for other addresses.
851  *
852  * We have to use BOP_ALLOC() method for C-OBP platforms because
853  * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
854  * sun4u platforms. On sun4u we flush tlb after trap table is taken
855  * over if we use large pages for kernel heap and kmem64. Since sun4u
856  * prom (unlike sun4v) calls va>tte-data first for client address
857  * translation prom's ttes for kmem64 can't get into TLB even if we
858  * later switch to prom's trap table again. C-OBP uses 4M pages for
859  * client mappings when possible so on all platforms we get the
860  * benefit from large mappings for kmem64 area immediately during
861  * boot.
862  *
863  * pseudo code:
864  * if (context != 0) {
865  *	return false
866  * } else if (miss_va in range[kmem64_base, kmem64_end)) {
867  *	tte = tte_template +
868  *		(((miss_va & pagemask) - kmem64_base));
869  *	return tte, true
870  * } else {
871  *	return pgmap@ result
872  * }
873  */
874 char kmem64_obp_str[] =
875 	"h# %lx constant kmem64-base "
876 	"h# %lx constant kmem64-end "
877 	"h# %lx constant kmem64-pagemask "
878 	"h# %lx constant kmem64-template "
879 
880 	": kmem64-tte ( addr cnum -- false | tte-data true ) "
881 	"    if                                       ( addr ) "
882 	"       drop false exit then                  ( false ) "
883 	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
884 	"	kmem64-pagemask and                   ( addr' ) "
885 	"	kmem64-base -                         ( addr' ) "
886 	"	kmem64-template +                     ( tte ) "
887 	"	true                                  ( tte true ) "
888 	"    else                                     ( addr ) "
889 	"	pgmap@                                ( tte ) "
890 	"       dup 0< if true else drop false then   ( tte true  |  false ) "
891 	"    then                                     ( tte true  |  false ) "
892 	"; "
893 
894 	"' kmem64-tte is va>tte-data "
895 ;
896 
897 static void
898 install_kmem64_tte()
899 {
900 	char b[sizeof (kmem64_obp_str) + (4 * 16)];
901 	tte_t tte;
902 
903 	PRM_DEBUG(kmem64_pabase);
904 	PRM_DEBUG(kmem64_szc);
905 	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
906 	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
907 	PRM_DEBUG(tte.ll);
908 	(void) sprintf(b, kmem64_obp_str,
909 	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
910 	ASSERT(strlen(b) < sizeof (b));
911 	prom_interpret(b, 0, 0, 0, 0, 0);
912 }
913 #endif	/* !C_OBP */
914 
915 /*
916  * As OBP takes up some RAM when the system boots, pages will already be "lost"
917  * to the system and reflected in npages by the time we see it.
918  *
919  * We only want to allocate kernel structures in the 64-bit virtual address
920  * space on systems with enough RAM to make the overhead of keeping track of
921  * an extra kernel memory segment worthwhile.
922  *
923  * Since OBP has already performed its memory allocations by this point, if we
924  * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
925  * memory in the 64-bit virtual address space; otherwise keep allocations
926  * contiguous with we've mapped so far in the 32-bit virtual address space.
927  */
928 #define	MINMOVE_RAM_MB	((size_t)1900)
929 #define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
930 #define	BYTES_TO_MB(b) ((b) / 1048576ul)
931 
932 pgcnt_t	tune_npages = (pgcnt_t)
933 	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
934 
935 #pragma weak page_set_colorequiv_arr_cpu
936 extern void page_set_colorequiv_arr_cpu(void);
937 extern void page_set_colorequiv_arr(void);
938 
939 static pgcnt_t ramdisk_npages;
940 static struct memlist *old_phys_avail;
941 
942 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
943 
944 static void
945 startup_memlist(void)
946 {
947 	size_t hmehash_sz, pagelist_sz, tt_sz;
948 	size_t psetable_sz;
949 	caddr_t alloc_base;
950 	caddr_t memspace;
951 	struct memlist *cur;
952 	size_t syslimit = (size_t)SYSLIMIT;
953 	size_t sysbase = (size_t)SYSBASE;
954 
955 	/*
956 	 * Initialize enough of the system to allow kmem_alloc to work by
957 	 * calling boot to allocate its memory until the time that
958 	 * kvm_init is completed.  The page structs are allocated after
959 	 * rounding up end to the nearest page boundary; the memsegs are
960 	 * initialized and the space they use comes from the kernel heap.
961 	 * With appropriate initialization, they can be reallocated later
962 	 * to a size appropriate for the machine's configuration.
963 	 *
964 	 * At this point, memory is allocated for things that will never
965 	 * need to be freed, this used to be "valloced".  This allows a
966 	 * savings as the pages don't need page structures to describe
967 	 * them because them will not be managed by the vm system.
968 	 */
969 
970 	/*
971 	 * We're loaded by boot with the following configuration (as
972 	 * specified in the sun4u/conf/Mapfile):
973 	 *
974 	 *	text:		4 MB chunk aligned on a 4MB boundary
975 	 *	data & bss:	4 MB chunk aligned on a 4MB boundary
976 	 *
977 	 * These two chunks will eventually be mapped by 2 locked 4MB
978 	 * ttes and will represent the nucleus of the kernel.  This gives
979 	 * us some free space that is already allocated, some or all of
980 	 * which is made available to kernel module text.
981 	 *
982 	 * The free space in the data-bss chunk is used for nucleus
983 	 * allocatable data structures and we reserve it using the
984 	 * nalloc_base and nalloc_end variables.  This space is currently
985 	 * being used for hat data structures required for tlb miss
986 	 * handling operations.  We align nalloc_base to a l2 cache
987 	 * linesize because this is the line size the hardware uses to
988 	 * maintain cache coherency.
989 	 * 512K is carved out for module data.
990 	 */
991 
992 	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
993 	e_moddata = moddata + MODDATA;
994 	nalloc_base = e_moddata;
995 
996 	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
997 	valloc_base = nalloc_base;
998 
999 	/*
1000 	 * Calculate the start of the data segment.
1001 	 */
1002 	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1003 		prom_panic("nucleus data overflow");
1004 
1005 	PRM_DEBUG(moddata);
1006 	PRM_DEBUG(nalloc_base);
1007 	PRM_DEBUG(nalloc_end);
1008 
1009 	/*
1010 	 * Remember any slop after e_text so we can give it to the modules.
1011 	 */
1012 	PRM_DEBUG(e_text);
1013 	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1014 	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1015 		prom_panic("nucleus text overflow");
1016 	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1017 	    modtext;
1018 	PRM_DEBUG(modtext);
1019 	PRM_DEBUG(modtext_sz);
1020 
1021 	init_boot_memlists();
1022 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1023 	    &boot_physavail, &boot_physavail_len,
1024 	    &boot_virtavail, &boot_virtavail_len);
1025 
1026 	/*
1027 	 * Remember what the physically available highest page is
1028 	 * so that dumpsys works properly, and find out how much
1029 	 * memory is installed.
1030 	 */
1031 	installed_top_size_memlist_array(boot_physinstalled,
1032 	    boot_physinstalled_len, &physmax, &physinstalled);
1033 	PRM_DEBUG(physinstalled);
1034 	PRM_DEBUG(physmax);
1035 
1036 	/* Fill out memory nodes config structure */
1037 	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1038 
1039 	/*
1040 	 * npages is the maximum of available physical memory possible.
1041 	 * (ie. it will never be more than this)
1042 	 *
1043 	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1044 	 * using phys_avail will underestimate what will end up being freed.
1045 	 * A better initial guess is just total memory minus the kernel text
1046 	 */
1047 	npages = physinstalled - btop(MMU_PAGESIZE4M);
1048 
1049 	/*
1050 	 * First allocate things that can go in the nucleus data page
1051 	 * (fault status, TSBs, dmv, CPUs)
1052 	 */
1053 	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1054 
1055 	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1056 		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1057 
1058 	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1059 		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1060 
1061 	if (ndata_alloc_dmv(&ndata) != 0)
1062 		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1063 
1064 	if (ndata_alloc_page_mutexs(&ndata) != 0)
1065 		cmn_err(CE_PANIC,
1066 		    "no more nucleus memory after page free lists alloc");
1067 
1068 	if (ndata_alloc_hat(&ndata) != 0)
1069 		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1070 
1071 	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1072 		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1073 
1074 	/*
1075 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1076 	 *
1077 	 * There are comments all over the SFMMU code warning of dire
1078 	 * consequences if the TSBs are moved out of 32-bit space.  This
1079 	 * is largely because the asm code uses "sethi %hi(addr)"-type
1080 	 * instructions which will not provide the expected result if the
1081 	 * address is a 64-bit one.
1082 	 *
1083 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1084 	 */
1085 	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1086 	PRM_DEBUG(alloc_base);
1087 
1088 	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1089 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1090 	PRM_DEBUG(alloc_base);
1091 
1092 	/*
1093 	 * Allocate IOMMU TSB array.  We do this here so that the physical
1094 	 * memory gets deducted from the PROM's physical memory list.
1095 	 */
1096 	alloc_base = iommu_tsb_init(alloc_base);
1097 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1098 	PRM_DEBUG(alloc_base);
1099 
1100 	/*
1101 	 * Allow for an early allocation of physically contiguous memory.
1102 	 */
1103 	alloc_base = contig_mem_prealloc(alloc_base, npages);
1104 
1105 	/*
1106 	 * Platforms like Starcat and OPL need special structures assigned in
1107 	 * 32-bit virtual address space because their probing routines execute
1108 	 * FCode, and FCode can't handle 64-bit virtual addresses...
1109 	 */
1110 	if (&plat_startup_memlist) {
1111 		alloc_base = plat_startup_memlist(alloc_base);
1112 		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1113 		    ecache_alignsize);
1114 		PRM_DEBUG(alloc_base);
1115 	}
1116 
1117 	/*
1118 	 * Save off where the contiguous allocations to date have ended
1119 	 * in econtig32.
1120 	 */
1121 	econtig32 = alloc_base;
1122 	PRM_DEBUG(econtig32);
1123 	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1124 		cmn_err(CE_PANIC, "econtig32 too big");
1125 
1126 	pp_sz = calc_pp_sz(npages);
1127 	PRM_DEBUG(pp_sz);
1128 	if (kpm_enable) {
1129 		kpm_pp_sz = calc_kpmpp_sz(npages);
1130 		PRM_DEBUG(kpm_pp_sz);
1131 	}
1132 
1133 	hmehash_sz = calc_hmehash_sz(npages);
1134 	PRM_DEBUG(hmehash_sz);
1135 
1136 	pagehash_sz = calc_pagehash_sz(npages);
1137 	PRM_DEBUG(pagehash_sz);
1138 
1139 	pagelist_sz = calc_free_pagelist_sz();
1140 	PRM_DEBUG(pagelist_sz);
1141 
1142 #ifdef	TRAPTRACE
1143 	tt_sz = calc_traptrace_sz();
1144 	PRM_DEBUG(tt_sz);
1145 #else
1146 	tt_sz = 0;
1147 #endif	/* TRAPTRACE */
1148 
1149 	/*
1150 	 * Place the array that protects pp->p_selock in the kmem64 wad.
1151 	 */
1152 	pse_shift = size_pse_array(npages, max_ncpus);
1153 	PRM_DEBUG(pse_shift);
1154 	pse_table_size = 1 << pse_shift;
1155 	PRM_DEBUG(pse_table_size);
1156 	psetable_sz = roundup(
1157 	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1158 	PRM_DEBUG(psetable_sz);
1159 
1160 	/*
1161 	 * Now allocate the whole wad
1162 	 */
1163 	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1164 	    pagelist_sz + tt_sz + psetable_sz;
1165 	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1166 	kmem64_base = (caddr_t)syslimit;
1167 	kmem64_end = kmem64_base + kmem64_sz;
1168 	if (alloc_kmem64(kmem64_base, kmem64_end)) {
1169 		/*
1170 		 * Attempt for kmem64 to allocate one big
1171 		 * contiguous chunk of memory failed.
1172 		 * We get here because we are sun4v.
1173 		 * We will proceed by breaking up
1174 		 * the allocation into two attempts.
1175 		 * First, we allocate kpm_pp_sz, hmehash_sz,
1176 		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1177 		 * one contiguous chunk. This is a much smaller
1178 		 * chunk and we should get it, if not we panic.
1179 		 * Note that hmehash and tt need to be physically
1180 		 * (in the real address sense) contiguous.
1181 		 * Next, we use bop_alloc_chunk() to
1182 		 * to allocate the page_t structures.
1183 		 * This will allow the page_t to be allocated
1184 		 * in multiple smaller chunks.
1185 		 * In doing so, the assumption that page_t is
1186 		 * physically contiguous no longer hold, this is ok
1187 		 * for sun4v but not for sun4u.
1188 		 */
1189 		size_t  tmp_size;
1190 		caddr_t tmp_base;
1191 
1192 		pp_sz  = roundup(pp_sz, PAGESIZE);
1193 
1194 		/*
1195 		 * Allocate kpm_pp_sz, hmehash_sz,
1196 		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1197 		 */
1198 		tmp_base = kmem64_base + pp_sz;
1199 		tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1200 		    pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1201 		if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1202 			prom_panic("kmem64 prom_alloc contig failed");
1203 		PRM_DEBUG(tmp_base);
1204 		PRM_DEBUG(tmp_size);
1205 
1206 		/*
1207 		 * Allocate the page_ts
1208 		 */
1209 		if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1210 			prom_panic("kmem64 bop_alloc_chunk page_t failed");
1211 		PRM_DEBUG(kmem64_base);
1212 		PRM_DEBUG(pp_sz);
1213 
1214 		kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1215 		ASSERT(kmem64_aligned_end >= kmem64_end);
1216 
1217 		kmem64_smchunks = 1;
1218 	} else {
1219 
1220 		/*
1221 		 * We need to adjust pp_sz for the normal
1222 		 * case where kmem64 can allocate one large chunk
1223 		 */
1224 		if (kpm_smallpages == 0) {
1225 			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1226 		} else {
1227 			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1228 			    sizeof (kpm_spage_t));
1229 		}
1230 		pp_sz = npages * sizeof (struct page);
1231 	}
1232 
1233 	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1234 		cmn_err(CE_PANIC, "not enough kmem64 space");
1235 	PRM_DEBUG(kmem64_base);
1236 	PRM_DEBUG(kmem64_end);
1237 	PRM_DEBUG(kmem64_aligned_end);
1238 
1239 	/*
1240 	 * ... and divy it up
1241 	 */
1242 	alloc_base = kmem64_base;
1243 
1244 	pp_base = (page_t *)alloc_base;
1245 	alloc_base += pp_sz;
1246 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1247 	PRM_DEBUG(pp_base);
1248 	PRM_DEBUG(npages);
1249 
1250 	if (kpm_enable) {
1251 		kpm_pp_base = alloc_base;
1252 		if (kpm_smallpages == 0) {
1253 			/* kpm_npages based on physinstalled, don't reset */
1254 			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1255 		} else {
1256 			kpm_npages = ptokpmpr(npages);
1257 			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1258 		}
1259 		alloc_base += kpm_pp_sz;
1260 		alloc_base =
1261 		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1262 		PRM_DEBUG(kpm_pp_base);
1263 	}
1264 
1265 	alloc_base = alloc_hmehash(alloc_base);
1266 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1267 	PRM_DEBUG(alloc_base);
1268 
1269 	page_hash = (page_t **)alloc_base;
1270 	alloc_base += pagehash_sz;
1271 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1272 	PRM_DEBUG(page_hash);
1273 
1274 	alloc_base = alloc_page_freelists(alloc_base);
1275 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1276 	PRM_DEBUG(alloc_base);
1277 
1278 #ifdef	TRAPTRACE
1279 	ttrace_buf = alloc_base;
1280 	alloc_base += tt_sz;
1281 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1282 	PRM_DEBUG(alloc_base);
1283 #endif	/* TRAPTRACE */
1284 
1285 	pse_mutex = (pad_mutex_t *)alloc_base;
1286 	alloc_base += psetable_sz;
1287 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1288 	PRM_DEBUG(alloc_base);
1289 
1290 	/*
1291 	 * Note that if we use small chunk allocations for
1292 	 * kmem64, we need to ensure kmem64_end is the same as
1293 	 * kmem64_aligned_end to prevent subsequent logic from
1294 	 * trying to reuse the overmapping.
1295 	 * Otherwise we adjust kmem64_end to what we really allocated.
1296 	 */
1297 	if (kmem64_smchunks) {
1298 		kmem64_end = kmem64_aligned_end;
1299 	} else {
1300 		kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1301 	}
1302 	kmem64_sz = kmem64_end - kmem64_base;
1303 
1304 	if (&ecache_init_scrub_flush_area) {
1305 		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1306 		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1307 	}
1308 
1309 	/*
1310 	 * If physmem is patched to be non-zero, use it instead of
1311 	 * the monitor value unless physmem is larger than the total
1312 	 * amount of memory on hand.
1313 	 */
1314 	if (physmem == 0 || physmem > npages)
1315 		physmem = npages;
1316 
1317 	/*
1318 	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1319 	 * is mounted as root. This memory is held down by OBP and unlike
1320 	 * the stub boot_archive is never released.
1321 	 *
1322 	 * In order to get things sized correctly on lower memory
1323 	 * machines (where the memory used by the ramdisk represents
1324 	 * a significant portion of memory), physmem is adjusted.
1325 	 *
1326 	 * This is done by subtracting the ramdisk_size which is set
1327 	 * to the size of the ramdisk (in Kb) in /etc/system at the
1328 	 * time the miniroot archive is constructed.
1329 	 */
1330 	if (root_is_ramdisk == B_TRUE) {
1331 		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1332 		physmem -= ramdisk_npages;
1333 	}
1334 
1335 	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1336 		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1337 
1338 	/*
1339 	 * Allocate space for the interrupt vector table.
1340 	 */
1341 	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1342 	if (memspace != (caddr_t)intr_vec_table)
1343 		prom_panic("interrupt vector table allocation failure");
1344 
1345 	/*
1346 	 * Between now and when we finish copying in the memory lists,
1347 	 * allocations happen so the space gets fragmented and the
1348 	 * lists longer.  Leave enough space for lists twice as
1349 	 * long as we have now; then roundup to a pagesize.
1350 	 */
1351 	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1352 	    prom_phys_avail_len() + prom_virt_avail_len());
1353 	memlist_sz *= 2;
1354 	memlist_sz = roundup(memlist_sz, PAGESIZE);
1355 	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1356 	if (memspace == NULL)
1357 		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1358 
1359 	memlist = (struct memlist *)memspace;
1360 	memlist_end = (char *)memspace + memlist_sz;
1361 	PRM_DEBUG(memlist);
1362 	PRM_DEBUG(memlist_end);
1363 
1364 	PRM_DEBUG(sysbase);
1365 	PRM_DEBUG(syslimit);
1366 	kernelheap_init((void *)sysbase, (void *)syslimit,
1367 	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1368 
1369 	/*
1370 	 * Take the most current snapshot we can by calling mem-update.
1371 	 */
1372 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1373 	    &boot_physavail, &boot_physavail_len,
1374 	    &boot_virtavail, &boot_virtavail_len);
1375 
1376 	/*
1377 	 * Remove the space used by prom_alloc from the kernel heap
1378 	 * plus the area actually used by the OBP (if any)
1379 	 * ignoring virtual addresses in virt_avail, above syslimit.
1380 	 */
1381 	virt_avail = memlist;
1382 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1383 
1384 	for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1385 		uint64_t range_base, range_size;
1386 
1387 		if ((range_base = cur->ml_address + cur->ml_size) <
1388 		    (uint64_t)sysbase)
1389 			continue;
1390 		if (range_base >= (uint64_t)syslimit)
1391 			break;
1392 		/*
1393 		 * Limit the range to end at syslimit.
1394 		 */
1395 		range_size = MIN(cur->ml_next->ml_address,
1396 		    (uint64_t)syslimit) - range_base;
1397 		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1398 		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1399 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1400 	}
1401 
1402 	phys_avail = memlist;
1403 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1404 
1405 	/*
1406 	 * Add any extra memory at the end of the ndata region if there's at
1407 	 * least a page to add.  There might be a few more pages available in
1408 	 * the middle of the ndata region, but for now they are ignored.
1409 	 */
1410 	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1411 	if (nalloc_base == NULL)
1412 		nalloc_base = nalloc_end;
1413 	ndata_remain_sz = nalloc_end - nalloc_base;
1414 
1415 	/*
1416 	 * Copy physinstalled list into kernel space.
1417 	 */
1418 	phys_install = memlist;
1419 	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1420 
1421 	/*
1422 	 * Create list of physical addrs we don't need pp's for:
1423 	 * kernel text 4M page
1424 	 * kernel data 4M page - ndata_remain_sz
1425 	 * kmem64 pages
1426 	 *
1427 	 * NB if adding any pages here, make sure no kpm page
1428 	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1429 	 */
1430 	nopp_list = memlist;
1431 	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1432 	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1433 	    &memlist, &nopp_list);
1434 
1435 	/* Don't add to nopp_list if kmem64 was allocated in smchunks */
1436 	if (!kmem64_smchunks)
1437 		memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1438 
1439 	if ((caddr_t)memlist > (memspace + memlist_sz))
1440 		prom_panic("memlist overflow");
1441 
1442 	/*
1443 	 * Size the pcf array based on the number of cpus in the box at
1444 	 * boot time.
1445 	 */
1446 	pcf_init();
1447 
1448 	/*
1449 	 * Initialize the page structures from the memory lists.
1450 	 */
1451 	kphysm_init();
1452 
1453 	availrmem_initial = availrmem = freemem;
1454 	PRM_DEBUG(availrmem);
1455 
1456 	/*
1457 	 * Some of the locks depend on page_hashsz being set!
1458 	 * kmem_init() depends on this; so, keep it here.
1459 	 */
1460 	page_lock_init();
1461 
1462 	/*
1463 	 * Initialize kernel memory allocator.
1464 	 */
1465 	kmem_init();
1466 
1467 	/*
1468 	 * Factor in colorequiv to check additional 'equivalent' bins
1469 	 */
1470 	if (&page_set_colorequiv_arr_cpu != NULL)
1471 		page_set_colorequiv_arr_cpu();
1472 	else
1473 		page_set_colorequiv_arr();
1474 
1475 	/*
1476 	 * Initialize bp_mapin().
1477 	 */
1478 	bp_init(shm_alignment, HAT_STRICTORDER);
1479 
1480 	/*
1481 	 * Reserve space for MPO mblock structs from the 32-bit heap.
1482 	 */
1483 
1484 	if (mpo_heap32_bufsz > (size_t)0) {
1485 		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1486 		    PAGESIZE, 0, 0, mpo_heap32_buf,
1487 		    mpo_heap32_buf + mpo_heap32_bufsz,
1488 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1489 	}
1490 	mem_config_init();
1491 }
1492 
1493 static void
1494 startup_modules(void)
1495 {
1496 	int nhblk1, nhblk8;
1497 	size_t  nhblksz;
1498 	pgcnt_t pages_per_hblk;
1499 	size_t hme8blk_sz, hme1blk_sz;
1500 
1501 	/*
1502 	 * The system file /etc/system was read already under startup_memlist.
1503 	 */
1504 	if (&set_platform_defaults)
1505 		set_platform_defaults();
1506 
1507 	/*
1508 	 * Calculate default settings of system parameters based upon
1509 	 * maxusers, yet allow to be overridden via the /etc/system file.
1510 	 */
1511 	param_calc(0);
1512 
1513 	mod_setup();
1514 
1515 	/*
1516 	 * Initialize system parameters
1517 	 */
1518 	param_init();
1519 
1520 	/*
1521 	 * maxmem is the amount of physical memory we're playing with.
1522 	 */
1523 	maxmem = physmem;
1524 
1525 	/* Set segkp limits. */
1526 	ncbase = kdi_segdebugbase;
1527 	ncend = kdi_segdebugbase;
1528 
1529 	/*
1530 	 * Initialize the hat layer.
1531 	 */
1532 	hat_init();
1533 
1534 	/*
1535 	 * Initialize segment management stuff.
1536 	 */
1537 	seg_init();
1538 
1539 	/*
1540 	 * Create the va>tte handler, so the prom can understand
1541 	 * kernel translations.  The handler is installed later, just
1542 	 * as we are about to take over the trap table from the prom.
1543 	 */
1544 	create_va_to_tte();
1545 
1546 	/*
1547 	 * Load the forthdebugger (optional)
1548 	 */
1549 	forthdebug_init();
1550 
1551 	/*
1552 	 * Create OBP node for console input callbacks
1553 	 * if it is needed.
1554 	 */
1555 	startup_create_io_node();
1556 
1557 	if (modloadonly("fs", "specfs") == -1)
1558 		halt("Can't load specfs");
1559 
1560 	if (modloadonly("fs", "devfs") == -1)
1561 		halt("Can't load devfs");
1562 
1563 	if (modloadonly("fs", "procfs") == -1)
1564 		halt("Can't load procfs");
1565 
1566 	if (modloadonly("misc", "swapgeneric") == -1)
1567 		halt("Can't load swapgeneric");
1568 
1569 	(void) modloadonly("sys", "lbl_edition");
1570 
1571 	dispinit();
1572 
1573 	/*
1574 	 * Infer meanings to the members of the idprom buffer.
1575 	 */
1576 	parse_idprom();
1577 
1578 	/* Read cluster configuration data. */
1579 	clconf_init();
1580 
1581 	setup_ddi();
1582 
1583 	/*
1584 	 * Lets take this opportunity to load the root device.
1585 	 */
1586 	if (loadrootmodules() != 0)
1587 		debug_enter("Can't load the root filesystem");
1588 
1589 	/*
1590 	 * Load tod driver module for the tod part found on this system.
1591 	 * Recompute the cpu frequency/delays based on tod as tod part
1592 	 * tends to keep time more accurately.
1593 	 */
1594 	if (&load_tod_module)
1595 		load_tod_module();
1596 
1597 	/*
1598 	 * Allow platforms to load modules which might
1599 	 * be needed after bootops are gone.
1600 	 */
1601 	if (&load_platform_modules)
1602 		load_platform_modules();
1603 
1604 	setcpudelay();
1605 
1606 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1607 	    &boot_physavail, &boot_physavail_len,
1608 	    &boot_virtavail, &boot_virtavail_len);
1609 
1610 	/*
1611 	 * Calculation and allocation of hmeblks needed to remap
1612 	 * the memory allocated by PROM till now.
1613 	 * Overestimate the number of hblk1 elements by assuming
1614 	 * worst case of TTE64K mappings.
1615 	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1616 	 */
1617 	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1618 	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1619 	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1620 	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1621 
1622 	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1623 
1624 	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1625 	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1626 	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1627 
1628 	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1629 
1630 	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1631 	nhblk8 = 0;
1632 	while (bop_alloc_pages > 1) {
1633 		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1634 		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1635 		bop_alloc_pages *= hme8blk_sz;
1636 		bop_alloc_pages = btopr(bop_alloc_pages);
1637 	}
1638 	nhblk8 += 2;
1639 
1640 	/*
1641 	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1642 	 * boundary, the number of hblk8's needed to map the entries in the
1643 	 * boot_virtavail list needs to be adjusted to take this into
1644 	 * consideration.  Thus, we need to add additional hblk8's since it
1645 	 * is possible that an hblk8 will not have all 8 slots used due to
1646 	 * alignment constraints.  Since there were boot_virtavail_len entries
1647 	 * in that list, we need to add that many hblk8's to the number
1648 	 * already calculated to make sure we don't underestimate.
1649 	 */
1650 	nhblk8 += boot_virtavail_len;
1651 	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1652 
1653 	/* Allocate in pagesize chunks */
1654 	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1655 	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1656 	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1657 }
1658 
1659 static void
1660 startup_bop_gone(void)
1661 {
1662 
1663 	/*
1664 	 * Destroy the MD initialized at startup
1665 	 * The startup initializes the MD framework
1666 	 * using prom and BOP alloc free it now.
1667 	 */
1668 	mach_descrip_startup_fini();
1669 
1670 	/*
1671 	 * We're done with prom allocations.
1672 	 */
1673 	bop_fini();
1674 
1675 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1676 	    &boot_physavail, &boot_physavail_len,
1677 	    &boot_virtavail, &boot_virtavail_len);
1678 
1679 	/*
1680 	 * setup physically contiguous area twice as large as the ecache.
1681 	 * this is used while doing displacement flush of ecaches
1682 	 */
1683 	if (&ecache_flush_address) {
1684 		ecache_flushaddr = ecache_flush_address();
1685 		if (ecache_flushaddr == (uint64_t)-1) {
1686 			cmn_err(CE_PANIC,
1687 			    "startup: no memory to set ecache_flushaddr");
1688 		}
1689 	}
1690 
1691 	/*
1692 	 * Virtual available next.
1693 	 */
1694 	ASSERT(virt_avail != NULL);
1695 	memlist_free_list(virt_avail);
1696 	virt_avail = memlist;
1697 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1698 
1699 }
1700 
1701 
1702 /*
1703  * startup_fixup_physavail - called from mach_sfmmu.c after the final
1704  * allocations have been performed.  We can't call it in startup_bop_gone
1705  * since later operations can cause obp to allocate more memory.
1706  */
1707 void
1708 startup_fixup_physavail(void)
1709 {
1710 	struct memlist *cur;
1711 	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1712 
1713 	PRM_DEBUG(kmem64_overmap_size);
1714 
1715 	/*
1716 	 * take the most current snapshot we can by calling mem-update
1717 	 */
1718 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1719 	    &boot_physavail, &boot_physavail_len,
1720 	    &boot_virtavail, &boot_virtavail_len);
1721 
1722 	/*
1723 	 * Copy phys_avail list, again.
1724 	 * Both the kernel/boot and the prom have been allocating
1725 	 * from the original list we copied earlier.
1726 	 */
1727 	cur = memlist;
1728 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1729 
1730 	/*
1731 	 * Add any unused kmem64 memory from overmapped page
1732 	 * (Note: va_to_pa does not work for kmem64_end)
1733 	 */
1734 	if (kmem64_overmap_size) {
1735 		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1736 		    kmem64_overmap_size, &memlist, &cur);
1737 	}
1738 
1739 	/*
1740 	 * Add any extra memory after e_data we added to the phys_avail list
1741 	 * back to the old list.
1742 	 */
1743 	if (ndata_remain_sz >= MMU_PAGESIZE)
1744 		memlist_add(va_to_pa(nalloc_base),
1745 		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1746 
1747 	/*
1748 	 * There isn't any bounds checking on the memlist area
1749 	 * so ensure it hasn't overgrown.
1750 	 */
1751 	if ((caddr_t)memlist > (caddr_t)memlist_end)
1752 		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1753 
1754 	/*
1755 	 * The kernel removes the pages that were allocated for it from
1756 	 * the freelist, but we now have to find any -extra- pages that
1757 	 * the prom has allocated for it's own book-keeping, and remove
1758 	 * them from the freelist too. sigh.
1759 	 */
1760 	sync_memlists(phys_avail, cur);
1761 
1762 	ASSERT(phys_avail != NULL);
1763 
1764 	old_phys_avail = phys_avail;
1765 	phys_avail = cur;
1766 }
1767 
1768 void
1769 update_kcage_ranges(uint64_t addr, uint64_t len)
1770 {
1771 	pfn_t base = btop(addr);
1772 	pgcnt_t num = btop(len);
1773 	int rv;
1774 
1775 	rv = kcage_range_add(base, num, kcage_startup_dir);
1776 
1777 	if (rv == ENOMEM) {
1778 		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1779 		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1780 	} else if (rv != 0) {
1781 		/* catch this in debug kernels */
1782 		ASSERT(0);
1783 
1784 		cmn_err(CE_WARN, "unexpected kcage_range_add"
1785 		    " return value %d", rv);
1786 	}
1787 }
1788 
1789 static void
1790 startup_vm(void)
1791 {
1792 	size_t	i;
1793 	struct segmap_crargs a;
1794 	struct segkpm_crargs b;
1795 
1796 	uint64_t avmem;
1797 	caddr_t va;
1798 	pgcnt_t	max_phys_segkp;
1799 	int	mnode;
1800 
1801 	extern int use_brk_lpg, use_stk_lpg;
1802 
1803 	/*
1804 	 * get prom's mappings, create hments for them and switch
1805 	 * to the kernel context.
1806 	 */
1807 	hat_kern_setup();
1808 
1809 	/*
1810 	 * Take over trap table
1811 	 */
1812 	setup_trap_table();
1813 
1814 	/*
1815 	 * Install the va>tte handler, so that the prom can handle
1816 	 * misses and understand the kernel table layout in case
1817 	 * we need call into the prom.
1818 	 */
1819 	install_va_to_tte();
1820 
1821 	/*
1822 	 * Set a flag to indicate that the tba has been taken over.
1823 	 */
1824 	tba_taken_over = 1;
1825 
1826 	/* initialize MMU primary context register */
1827 	mmu_init_kcontext();
1828 
1829 	/*
1830 	 * The boot cpu can now take interrupts, x-calls, x-traps
1831 	 */
1832 	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1833 	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1834 
1835 	/*
1836 	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1837 	 */
1838 	tbr_wr_addr_inited = 1;
1839 
1840 	/*
1841 	 * Initialize VM system, and map kernel address space.
1842 	 */
1843 	kvm_init();
1844 
1845 	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1846 	if (kernel_cage_enable) {
1847 		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1848 	}
1849 	memlist_free_list(old_phys_avail);
1850 
1851 	/*
1852 	 * If the following is true, someone has patched
1853 	 * phsymem to be less than the number of pages that
1854 	 * the system actually has.  Remove pages until system
1855 	 * memory is limited to the requested amount.  Since we
1856 	 * have allocated page structures for all pages, we
1857 	 * correct the amount of memory we want to remove
1858 	 * by the size of the memory used to hold page structures
1859 	 * for the non-used pages.
1860 	 */
1861 	if (physmem + ramdisk_npages < npages) {
1862 		pgcnt_t diff, off;
1863 		struct page *pp;
1864 		struct seg kseg;
1865 
1866 		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1867 
1868 		off = 0;
1869 		diff = npages - (physmem + ramdisk_npages);
1870 		diff -= mmu_btopr(diff * sizeof (struct page));
1871 		kseg.s_as = &kas;
1872 		while (diff--) {
1873 			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1874 			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1875 			    &kseg, (caddr_t)off);
1876 			if (pp == NULL)
1877 				cmn_err(CE_PANIC, "limited physmem too much!");
1878 			page_io_unlock(pp);
1879 			page_downgrade(pp);
1880 			availrmem--;
1881 			off += MMU_PAGESIZE;
1882 		}
1883 	}
1884 
1885 	/*
1886 	 * When printing memory, show the total as physmem less
1887 	 * that stolen by a debugger.
1888 	 */
1889 	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1890 	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1891 	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1892 
1893 	avmem = (uint64_t)freemem << PAGESHIFT;
1894 	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1895 
1896 	/*
1897 	 * For small memory systems disable automatic large pages.
1898 	 */
1899 	if (physmem < privm_lpg_min_physmem) {
1900 		use_brk_lpg = 0;
1901 		use_stk_lpg = 0;
1902 	}
1903 
1904 	/*
1905 	 * Perform platform specific freelist processing
1906 	 */
1907 	if (&plat_freelist_process) {
1908 		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1909 			if (mem_node_config[mnode].exists)
1910 				plat_freelist_process(mnode);
1911 	}
1912 
1913 	/*
1914 	 * Initialize the segkp segment type.  We position it
1915 	 * after the configured tables and buffers (whose end
1916 	 * is given by econtig) and before V_WKBASE_ADDR.
1917 	 * Also in this area is segkmap (size SEGMAPSIZE).
1918 	 */
1919 
1920 	/* XXX - cache alignment? */
1921 	va = (caddr_t)SEGKPBASE;
1922 	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1923 
1924 	max_phys_segkp = (physmem * 2);
1925 
1926 	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1927 		segkpsize = btop(SEGKPDEFSIZE);
1928 		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1929 		    "segkpsize has been reset to %ld pages", segkpsize);
1930 	}
1931 
1932 	i = ptob(MIN(segkpsize, max_phys_segkp));
1933 
1934 	rw_enter(&kas.a_lock, RW_WRITER);
1935 	if (seg_attach(&kas, va, i, segkp) < 0)
1936 		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1937 	if (segkp_create(segkp) != 0)
1938 		cmn_err(CE_PANIC, "startup: segkp_create failed");
1939 	rw_exit(&kas.a_lock);
1940 
1941 	/*
1942 	 * kpm segment
1943 	 */
1944 	segmap_kpm = kpm_enable &&
1945 	    segmap_kpm && PAGESIZE == MAXBSIZE;
1946 
1947 	if (kpm_enable) {
1948 		rw_enter(&kas.a_lock, RW_WRITER);
1949 
1950 		/*
1951 		 * The segkpm virtual range range is larger than the
1952 		 * actual physical memory size and also covers gaps in
1953 		 * the physical address range for the following reasons:
1954 		 * . keep conversion between segkpm and physical addresses
1955 		 *   simple, cheap and unambiguous.
1956 		 * . avoid extension/shrink of the the segkpm in case of DR.
1957 		 * . avoid complexity for handling of virtual addressed
1958 		 *   caches, segkpm and the regular mapping scheme must be
1959 		 *   kept in sync wrt. the virtual color of mapped pages.
1960 		 * Any accesses to virtual segkpm ranges not backed by
1961 		 * physical memory will fall through the memseg pfn hash
1962 		 * and will be handled in segkpm_fault.
1963 		 * Additional kpm_size spaces needed for vac alias prevention.
1964 		 */
1965 		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1966 		    segkpm) < 0)
1967 			cmn_err(CE_PANIC, "cannot attach segkpm");
1968 
1969 		b.prot = PROT_READ | PROT_WRITE;
1970 		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1971 
1972 		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1973 			panic("segkpm_create segkpm");
1974 
1975 		rw_exit(&kas.a_lock);
1976 
1977 		mach_kpm_init();
1978 	}
1979 
1980 	va = kpm_vbase + (kpm_size * vac_colors);
1981 
1982 	if (!segzio_fromheap) {
1983 		size_t size;
1984 		size_t physmem_b = mmu_ptob(physmem);
1985 
1986 		/* size is in bytes, segziosize is in pages */
1987 		if (segziosize == 0) {
1988 			size = physmem_b;
1989 		} else {
1990 			size = mmu_ptob(segziosize);
1991 		}
1992 
1993 		if (size < SEGZIOMINSIZE) {
1994 			size = SEGZIOMINSIZE;
1995 		} else if (size > SEGZIOMAXSIZE) {
1996 			size = SEGZIOMAXSIZE;
1997 			/*
1998 			 * On 64-bit x86, we only have 2TB of KVA.  This exists
1999 			 * for parity with x86.
2000 			 *
2001 			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2002 			 * doesn't consume all of KVA.  However, if we have a
2003 			 * system that has more thant 512gb of physical memory,
2004 			 * we can actually consume about half of the difference
2005 			 * between 512gb and the rest of the available physical
2006 			 * memory.
2007 			 */
2008 			if (physmem_b > SEGZIOMAXSIZE) {
2009 				size += (physmem_b - SEGZIOMAXSIZE) / 2;
2010 		}
2011 		}
2012 		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2013 		/* put the base of the ZIO segment after the kpm segment */
2014 		segzio_base = va;
2015 		va += mmu_ptob(segziosize);
2016 		PRM_DEBUG(segziosize);
2017 		PRM_DEBUG(segzio_base);
2018 
2019 		/*
2020 		 * On some platforms, kvm_init is called after the kpm
2021 		 * sizes have been determined.  On SPARC, kvm_init is called
2022 		 * before, so we have to attach the kzioseg after kvm is
2023 		 * initialized, otherwise we'll try to allocate from the boot
2024 		 * area since the kernel heap hasn't yet been configured.
2025 		 */
2026 		rw_enter(&kas.a_lock, RW_WRITER);
2027 
2028 		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2029 		    &kzioseg);
2030 		(void) segkmem_create(&kzioseg);
2031 
2032 		/* create zio area covering new segment */
2033 		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2034 
2035 		rw_exit(&kas.a_lock);
2036 	}
2037 
2038 	if (ppvm_enable) {
2039 		caddr_t ppvm_max;
2040 
2041 		/*
2042 		 * ppvm refers to the static VA space used to map
2043 		 * the page_t's for dynamically added memory.
2044 		 *
2045 		 * ppvm_base should not cross a potential VA hole.
2046 		 *
2047 		 * ppvm_size should be large enough to map the
2048 		 * page_t's needed to manage all of KPM range.
2049 		 */
2050 		ppvm_size =
2051 		    roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2052 		    MMU_PAGESIZE);
2053 		ppvm_max = (caddr_t)(0ull - ppvm_size);
2054 		ppvm_base = (page_t *)va;
2055 
2056 		if ((caddr_t)ppvm_base <= hole_end) {
2057 			cmn_err(CE_WARN,
2058 			    "Memory DR disabled: invalid DR map base: 0x%p\n",
2059 			    (void *)ppvm_base);
2060 			ppvm_enable = 0;
2061 		} else if ((caddr_t)ppvm_base > ppvm_max) {
2062 			uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2063 
2064 			cmn_err(CE_WARN,
2065 			    "Memory DR disabled: insufficient DR map size:"
2066 			    " 0x%lx (needed 0x%lx)\n",
2067 			    ppvm_size - diff, ppvm_size);
2068 			ppvm_enable = 0;
2069 		}
2070 		PRM_DEBUG(ppvm_size);
2071 		PRM_DEBUG(ppvm_base);
2072 	}
2073 
2074 	/*
2075 	 * Now create generic mapping segment.  This mapping
2076 	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2077 	 * virtual address is greater than the amount of free
2078 	 * memory that is available, then we trim back the
2079 	 * segment size to that amount
2080 	 */
2081 	va = (caddr_t)SEGMAPBASE;
2082 
2083 	/*
2084 	 * 1201049: segkmap base address must be MAXBSIZE aligned
2085 	 */
2086 	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2087 
2088 	/*
2089 	 * Set size of segmap to percentage of freemem at boot,
2090 	 * but stay within the allowable range
2091 	 * Note we take percentage  before converting from pages
2092 	 * to bytes to avoid an overflow on 32-bit kernels.
2093 	 */
2094 	i = mmu_ptob((freemem * segmap_percent) / 100);
2095 
2096 	if (i < MINMAPSIZE)
2097 		i = MINMAPSIZE;
2098 
2099 	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2100 		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2101 
2102 	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
2103 
2104 	rw_enter(&kas.a_lock, RW_WRITER);
2105 	if (seg_attach(&kas, va, i, segkmap) < 0)
2106 		cmn_err(CE_PANIC, "cannot attach segkmap");
2107 
2108 	a.prot = PROT_READ | PROT_WRITE;
2109 	a.shmsize = shm_alignment;
2110 	a.nfreelist = 0;	/* use segmap driver defaults */
2111 
2112 	if (segmap_create(segkmap, (caddr_t)&a) != 0)
2113 		panic("segmap_create segkmap");
2114 	rw_exit(&kas.a_lock);
2115 
2116 	segdev_init();
2117 }
2118 
2119 static void
2120 startup_end(void)
2121 {
2122 	if ((caddr_t)memlist > (caddr_t)memlist_end)
2123 		panic("memlist overflow 2");
2124 	memlist_free_block((caddr_t)memlist,
2125 	    ((caddr_t)memlist_end - (caddr_t)memlist));
2126 	memlist = NULL;
2127 
2128 	/* enable page_relocation since OBP is now done */
2129 	page_relocate_ready = 1;
2130 
2131 	/*
2132 	 * Perform tasks that get done after most of the VM
2133 	 * initialization has been done but before the clock
2134 	 * and other devices get started.
2135 	 */
2136 	kern_setup1();
2137 
2138 	/*
2139 	 * Perform CPC initialization for this CPU.
2140 	 */
2141 	kcpc_hw_init();
2142 
2143 	/*
2144 	 * Intialize the VM arenas for allocating physically
2145 	 * contiguus memory chunk for interrupt queues snd
2146 	 * allocate/register boot cpu's queues, if any and
2147 	 * allocate dump buffer for sun4v systems to store
2148 	 * extra crash information during crash dump
2149 	 */
2150 	contig_mem_init();
2151 	mach_descrip_init();
2152 
2153 	if (cpu_intrq_setup(CPU)) {
2154 		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2155 	}
2156 	cpu_intrq_register(CPU);
2157 	mach_htraptrace_setup(CPU->cpu_id);
2158 	mach_htraptrace_configure(CPU->cpu_id);
2159 	mach_dump_buffer_init();
2160 
2161 	/*
2162 	 * Initialize interrupt related stuff
2163 	 */
2164 	cpu_intr_alloc(CPU, NINTR_THREADS);
2165 
2166 	(void) splzs();			/* allow hi clock ints but not zs */
2167 
2168 	/*
2169 	 * Initialize errors.
2170 	 */
2171 	error_init();
2172 
2173 	/*
2174 	 * Note that we may have already used kernel bcopy before this
2175 	 * point - but if you really care about this, adb the use_hw_*
2176 	 * variables to 0 before rebooting.
2177 	 */
2178 	mach_hw_copy_limit();
2179 
2180 	/*
2181 	 * Install the "real" preemption guards before DDI services
2182 	 * are available.
2183 	 */
2184 	(void) prom_set_preprom(kern_preprom);
2185 	(void) prom_set_postprom(kern_postprom);
2186 	CPU->cpu_m.mutex_ready = 1;
2187 
2188 	/*
2189 	 * Initialize segnf (kernel support for non-faulting loads).
2190 	 */
2191 	segnf_init();
2192 
2193 	/*
2194 	 * Configure the root devinfo node.
2195 	 */
2196 	configure();		/* set up devices */
2197 	mach_cpu_halt_idle();
2198 }
2199 
2200 
2201 void
2202 post_startup(void)
2203 {
2204 #ifdef	PTL1_PANIC_DEBUG
2205 	extern void init_ptl1_thread(void);
2206 #endif	/* PTL1_PANIC_DEBUG */
2207 	extern void abort_sequence_init(void);
2208 
2209 	/*
2210 	 * Set the system wide, processor-specific flags to be passed
2211 	 * to userland via the aux vector for performance hints and
2212 	 * instruction set extensions.
2213 	 */
2214 	bind_hwcap();
2215 
2216 	/*
2217 	 * Startup memory scrubber (if any)
2218 	 */
2219 	mach_memscrub();
2220 
2221 	/*
2222 	 * Allocate soft interrupt to handle abort sequence.
2223 	 */
2224 	abort_sequence_init();
2225 
2226 	/*
2227 	 * Configure the rest of the system.
2228 	 * Perform forceloading tasks for /etc/system.
2229 	 */
2230 	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2231 	/*
2232 	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2233 	 * ON4.0: This must be fixed or restated in /etc/systems.
2234 	 */
2235 	(void) modload("fs", "procfs");
2236 
2237 	/* load machine class specific drivers */
2238 	load_mach_drivers();
2239 
2240 	/* load platform specific drivers */
2241 	if (&load_platform_drivers)
2242 		load_platform_drivers();
2243 
2244 	/* load vis simulation module, if we are running w/fpu off */
2245 	if (!fpu_exists) {
2246 		if (modload("misc", "vis") == -1)
2247 			halt("Can't load vis");
2248 	}
2249 
2250 	mach_fpras();
2251 
2252 	maxmem = freemem;
2253 
2254 	pg_init();
2255 
2256 #ifdef	PTL1_PANIC_DEBUG
2257 	init_ptl1_thread();
2258 #endif	/* PTL1_PANIC_DEBUG */
2259 }
2260 
2261 #ifdef	PTL1_PANIC_DEBUG
2262 int		ptl1_panic_test = 0;
2263 int		ptl1_panic_xc_one_test = 0;
2264 int		ptl1_panic_xc_all_test = 0;
2265 int		ptl1_panic_xt_one_test = 0;
2266 int		ptl1_panic_xt_all_test = 0;
2267 kthread_id_t	ptl1_thread_p = NULL;
2268 kcondvar_t	ptl1_cv;
2269 kmutex_t	ptl1_mutex;
2270 int		ptl1_recurse_count_threshold = 0x40;
2271 int		ptl1_recurse_trap_threshold = 0x3d;
2272 extern void	ptl1_recurse(int, int);
2273 extern void	ptl1_panic_xt(int, int);
2274 
2275 /*
2276  * Called once per second by timeout() to wake up
2277  * the ptl1_panic thread to see if it should cause
2278  * a trap to the ptl1_panic() code.
2279  */
2280 /* ARGSUSED */
2281 static void
2282 ptl1_wakeup(void *arg)
2283 {
2284 	mutex_enter(&ptl1_mutex);
2285 	cv_signal(&ptl1_cv);
2286 	mutex_exit(&ptl1_mutex);
2287 }
2288 
2289 /*
2290  * ptl1_panic cross call function:
2291  *     Needed because xc_one() and xc_some() can pass
2292  *	64 bit args but ptl1_recurse() expects ints.
2293  */
2294 static void
2295 ptl1_panic_xc(void)
2296 {
2297 	ptl1_recurse(ptl1_recurse_count_threshold,
2298 	    ptl1_recurse_trap_threshold);
2299 }
2300 
2301 /*
2302  * The ptl1 thread waits for a global flag to be set
2303  * and uses the recurse thresholds to set the stack depth
2304  * to cause a ptl1_panic() directly via a call to ptl1_recurse
2305  * or indirectly via the cross call and cross trap functions.
2306  *
2307  * This is useful testing stack overflows and normal
2308  * ptl1_panic() states with a know stack frame.
2309  *
2310  * ptl1_recurse() is an asm function in ptl1_panic.s that
2311  * sets the {In, Local, Out, and Global} registers to a
2312  * know state on the stack and just prior to causing a
2313  * test ptl1_panic trap.
2314  */
2315 static void
2316 ptl1_thread(void)
2317 {
2318 	mutex_enter(&ptl1_mutex);
2319 	while (ptl1_thread_p) {
2320 		cpuset_t	other_cpus;
2321 		int		cpu_id;
2322 		int		my_cpu_id;
2323 		int		target_cpu_id;
2324 		int		target_found;
2325 
2326 		if (ptl1_panic_test) {
2327 			ptl1_recurse(ptl1_recurse_count_threshold,
2328 			    ptl1_recurse_trap_threshold);
2329 		}
2330 
2331 		/*
2332 		 * Find potential targets for x-call and x-trap,
2333 		 * if any exist while preempt is disabled we
2334 		 * start a ptl1_panic if requested via a
2335 		 * globals.
2336 		 */
2337 		kpreempt_disable();
2338 		my_cpu_id = CPU->cpu_id;
2339 		other_cpus = cpu_ready_set;
2340 		CPUSET_DEL(other_cpus, CPU->cpu_id);
2341 		target_found = 0;
2342 		if (!CPUSET_ISNULL(other_cpus)) {
2343 			/*
2344 			 * Pick the first one
2345 			 */
2346 			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2347 				if (cpu_id == my_cpu_id)
2348 					continue;
2349 
2350 				if (CPU_XCALL_READY(cpu_id)) {
2351 					target_cpu_id = cpu_id;
2352 					target_found = 1;
2353 					break;
2354 				}
2355 			}
2356 			ASSERT(target_found);
2357 
2358 			if (ptl1_panic_xc_one_test) {
2359 				xc_one(target_cpu_id,
2360 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2361 			}
2362 			if (ptl1_panic_xc_all_test) {
2363 				xc_some(other_cpus,
2364 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2365 			}
2366 			if (ptl1_panic_xt_one_test) {
2367 				xt_one(target_cpu_id,
2368 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2369 			}
2370 			if (ptl1_panic_xt_all_test) {
2371 				xt_some(other_cpus,
2372 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2373 			}
2374 		}
2375 		kpreempt_enable();
2376 		(void) timeout(ptl1_wakeup, NULL, hz);
2377 		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2378 	}
2379 	mutex_exit(&ptl1_mutex);
2380 }
2381 
2382 /*
2383  * Called during early startup to create the ptl1_thread
2384  */
2385 void
2386 init_ptl1_thread(void)
2387 {
2388 	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2389 	    &p0, TS_RUN, 0);
2390 }
2391 #endif	/* PTL1_PANIC_DEBUG */
2392 
2393 
2394 static void
2395 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2396 {
2397 	struct memlist *new;
2398 
2399 	new = *memlistp;
2400 	new->ml_address = start;
2401 	new->ml_size = len;
2402 	*memlistp = new + 1;
2403 }
2404 
2405 /*
2406  * Add to a memory list.
2407  * start = start of new memory segment
2408  * len = length of new memory segment in bytes
2409  * memlistp = pointer to array of available memory segment structures
2410  * curmemlistp = memory list to which to add segment.
2411  */
2412 static void
2413 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2414     struct memlist **curmemlistp)
2415 {
2416 	struct memlist *new = *memlistp;
2417 
2418 	memlist_new(start, len, memlistp);
2419 	memlist_insert(new, curmemlistp);
2420 }
2421 
2422 static int
2423 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2424 {
2425 	int nseg;
2426 	size_t memseg_sz;
2427 	struct memseg *msp;
2428 
2429 	/*
2430 	 * The memseg list is for the chunks of physical memory that
2431 	 * will be managed by the vm system.  The number calculated is
2432 	 * a guess as boot may fragment it more when memory allocations
2433 	 * are made before kphysm_init().
2434 	 */
2435 	memseg_sz = (avail + 10) * sizeof (struct memseg);
2436 	memseg_sz = roundup(memseg_sz, PAGESIZE);
2437 	nseg = memseg_sz / sizeof (struct memseg);
2438 	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2439 	if (msp == NULL)
2440 		return (1);
2441 	PRM_DEBUG(memseg_free);
2442 
2443 	while (nseg--) {
2444 		msp->next = memseg_free;
2445 		memseg_free = msp;
2446 		msp++;
2447 	}
2448 	return (0);
2449 }
2450 
2451 /*
2452  * In the case of architectures that support dynamic addition of
2453  * memory at run-time there are two cases where memsegs need to
2454  * be initialized and added to the memseg list.
2455  * 1) memsegs that are constructed at startup.
2456  * 2) memsegs that are constructed at run-time on
2457  *    hot-plug capable architectures.
2458  * This code was originally part of the function kphysm_init().
2459  */
2460 
2461 static void
2462 memseg_list_add(struct memseg *memsegp)
2463 {
2464 	struct memseg **prev_memsegp;
2465 	pgcnt_t num;
2466 
2467 	/* insert in memseg list, decreasing number of pages order */
2468 
2469 	num = MSEG_NPAGES(memsegp);
2470 
2471 	for (prev_memsegp = &memsegs; *prev_memsegp;
2472 	    prev_memsegp = &((*prev_memsegp)->next)) {
2473 		if (num > MSEG_NPAGES(*prev_memsegp))
2474 			break;
2475 	}
2476 
2477 	memsegp->next = *prev_memsegp;
2478 	*prev_memsegp = memsegp;
2479 
2480 	if (kpm_enable) {
2481 		memsegp->nextpa = (memsegp->next) ?
2482 		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2483 
2484 		if (prev_memsegp != &memsegs) {
2485 			struct memseg *msp;
2486 			msp = (struct memseg *)((caddr_t)prev_memsegp -
2487 			    offsetof(struct memseg, next));
2488 			msp->nextpa = va_to_pa(memsegp);
2489 		} else {
2490 			memsegspa = va_to_pa(memsegs);
2491 		}
2492 	}
2493 }
2494 
2495 /*
2496  * PSM add_physmem_cb(). US-II and newer processors have some
2497  * flavor of the prefetch capability implemented. We exploit
2498  * this capability for optimum performance.
2499  */
2500 #define	PREFETCH_BYTES	64
2501 
2502 void
2503 add_physmem_cb(page_t *pp, pfn_t pnum)
2504 {
2505 	extern void	 prefetch_page_w(void *);
2506 
2507 	pp->p_pagenum = pnum;
2508 
2509 	/*
2510 	 * Prefetch one more page_t into E$. To prevent future
2511 	 * mishaps with the sizeof(page_t) changing on us, we
2512 	 * catch this on debug kernels if we can't bring in the
2513 	 * entire hpage with 2 PREFETCH_BYTES reads. See
2514 	 * also, sun4u/cpu/cpu_module.c
2515 	 */
2516 	/*LINTED*/
2517 	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2518 	prefetch_page_w((char *)pp);
2519 }
2520 
2521 /*
2522  * Find memseg with given pfn
2523  */
2524 static struct memseg *
2525 memseg_find(pfn_t base, pfn_t *next)
2526 {
2527 	struct memseg *seg;
2528 
2529 	if (next != NULL)
2530 		*next = LONG_MAX;
2531 	for (seg = memsegs; seg != NULL; seg = seg->next) {
2532 		if (base >= seg->pages_base && base < seg->pages_end)
2533 			return (seg);
2534 		if (next != NULL && seg->pages_base > base &&
2535 		    seg->pages_base < *next)
2536 			*next = seg->pages_base;
2537 	}
2538 	return (NULL);
2539 }
2540 
2541 /*
2542  * Put page allocated by OBP on prom_ppages
2543  */
2544 static void
2545 kphysm_erase(uint64_t addr, uint64_t len)
2546 {
2547 	struct page *pp;
2548 	struct memseg *seg;
2549 	pfn_t base = btop(addr), next;
2550 	pgcnt_t num = btop(len);
2551 
2552 	while (num != 0) {
2553 		pgcnt_t off, left;
2554 
2555 		seg = memseg_find(base, &next);
2556 		if (seg == NULL) {
2557 			if (next == LONG_MAX)
2558 				break;
2559 			left = MIN(next - base, num);
2560 			base += left, num -= left;
2561 			continue;
2562 		}
2563 		off = base - seg->pages_base;
2564 		pp = seg->pages + off;
2565 		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2566 		while (num != left) {
2567 			/*
2568 			 * init it, lock it, and hashin on prom_pages vp.
2569 			 *
2570 			 * Mark it as NONRELOC to let DR know the page
2571 			 * is locked long term, otherwise DR hangs when
2572 			 * trying to remove those pages.
2573 			 *
2574 			 * XXX	vnode offsets on the prom_ppages vnode
2575 			 *	are page numbers (gack) for >32 bit
2576 			 *	physical memory machines.
2577 			 */
2578 			PP_SETNORELOC(pp);
2579 			add_physmem_cb(pp, base);
2580 			if (page_trylock(pp, SE_EXCL) == 0)
2581 				cmn_err(CE_PANIC, "prom page locked");
2582 			(void) page_hashin(pp, &promvp,
2583 			    (offset_t)base, NULL);
2584 			(void) page_pp_lock(pp, 0, 1);
2585 			pp++, base++, num--;
2586 		}
2587 	}
2588 }
2589 
2590 static page_t *ppnext;
2591 static pgcnt_t ppleft;
2592 
2593 static void *kpm_ppnext;
2594 static pgcnt_t kpm_ppleft;
2595 
2596 /*
2597  * Create a memseg
2598  */
2599 static void
2600 kphysm_memseg(uint64_t addr, uint64_t len)
2601 {
2602 	pfn_t base = btop(addr);
2603 	pgcnt_t num = btop(len);
2604 	struct memseg *seg;
2605 
2606 	seg = memseg_free;
2607 	memseg_free = seg->next;
2608 	ASSERT(seg != NULL);
2609 
2610 	seg->pages = ppnext;
2611 	seg->epages = ppnext + num;
2612 	seg->pages_base = base;
2613 	seg->pages_end = base + num;
2614 	ppnext += num;
2615 	ppleft -= num;
2616 
2617 	if (kpm_enable) {
2618 		pgcnt_t kpnum = ptokpmpr(num);
2619 
2620 		if (kpnum > kpm_ppleft)
2621 			panic("kphysm_memseg: kpm_pp overflow");
2622 		seg->pagespa = va_to_pa(seg->pages);
2623 		seg->epagespa = va_to_pa(seg->epages);
2624 		seg->kpm_pbase = kpmptop(ptokpmp(base));
2625 		seg->kpm_nkpmpgs = kpnum;
2626 		/*
2627 		 * In the kpm_smallpage case, the kpm array
2628 		 * is 1-1 wrt the page array
2629 		 */
2630 		if (kpm_smallpages) {
2631 			kpm_spage_t *kpm_pp = kpm_ppnext;
2632 
2633 			kpm_ppnext = kpm_pp + kpnum;
2634 			seg->kpm_spages = kpm_pp;
2635 			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2636 		} else {
2637 			kpm_page_t *kpm_pp = kpm_ppnext;
2638 
2639 			kpm_ppnext = kpm_pp + kpnum;
2640 			seg->kpm_pages = kpm_pp;
2641 			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2642 			/* ASSERT no kpm overlaps */
2643 			ASSERT(
2644 			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2645 			ASSERT(memseg_find(
2646 			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2647 		}
2648 		kpm_ppleft -= kpnum;
2649 	}
2650 
2651 	memseg_list_add(seg);
2652 }
2653 
2654 /*
2655  * Add range to free list
2656  */
2657 void
2658 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2659 {
2660 	struct page *pp;
2661 	struct memseg *seg;
2662 	pfn_t base = btop(addr);
2663 	pgcnt_t num = btop(len);
2664 
2665 	seg = memseg_find(base, NULL);
2666 	ASSERT(seg != NULL);
2667 	pp = seg->pages + (base - seg->pages_base);
2668 
2669 	if (reclaim) {
2670 		struct page *rpp = pp;
2671 		struct page *lpp = pp + num;
2672 
2673 		/*
2674 		 * page should be locked on prom_ppages
2675 		 * unhash and unlock it
2676 		 */
2677 		while (rpp < lpp) {
2678 			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2679 			ASSERT(PP_ISNORELOC(rpp));
2680 			PP_CLRNORELOC(rpp);
2681 			page_pp_unlock(rpp, 0, 1);
2682 			page_hashout(rpp, NULL);
2683 			page_unlock(rpp);
2684 			rpp++;
2685 		}
2686 	}
2687 
2688 	/*
2689 	 * add_physmem() initializes the PSM part of the page
2690 	 * struct by calling the PSM back with add_physmem_cb().
2691 	 * In addition it coalesces pages into larger pages as
2692 	 * it initializes them.
2693 	 */
2694 	add_physmem(pp, num, base);
2695 }
2696 
2697 /*
2698  * kphysm_init() tackles the problem of initializing physical memory.
2699  */
2700 static void
2701 kphysm_init(void)
2702 {
2703 	struct memlist *pmem;
2704 
2705 	ASSERT(page_hash != NULL && page_hashsz != 0);
2706 
2707 	ppnext = pp_base;
2708 	ppleft = npages;
2709 	kpm_ppnext = kpm_pp_base;
2710 	kpm_ppleft = kpm_npages;
2711 
2712 	/*
2713 	 * installed pages not on nopp_memlist go in memseg list
2714 	 */
2715 	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2716 
2717 	/*
2718 	 * Free the avail list
2719 	 */
2720 	for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2721 		kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2722 
2723 	/*
2724 	 * Erase pages that aren't available
2725 	 */
2726 	diff_memlists(phys_install, phys_avail, kphysm_erase);
2727 
2728 	build_pfn_hash();
2729 }
2730 
2731 /*
2732  * Kernel VM initialization.
2733  * Assumptions about kernel address space ordering:
2734  *	(1) gap (user space)
2735  *	(2) kernel text
2736  *	(3) kernel data/bss
2737  *	(4) gap
2738  *	(5) kernel data structures
2739  *	(6) gap
2740  *	(7) debugger (optional)
2741  *	(8) monitor
2742  *	(9) gap (possibly null)
2743  *	(10) dvma
2744  *	(11) devices
2745  */
2746 static void
2747 kvm_init(void)
2748 {
2749 	/*
2750 	 * Put the kernel segments in kernel address space.
2751 	 */
2752 	rw_enter(&kas.a_lock, RW_WRITER);
2753 	as_avlinit(&kas);
2754 
2755 	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2756 	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2757 	(void) segkmem_create(&ktextseg);
2758 
2759 	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2760 	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2761 	(void) segkmem_create(&ktexthole);
2762 
2763 	(void) seg_attach(&kas, (caddr_t)valloc_base,
2764 	    (size_t)(econtig32 - valloc_base), &kvalloc);
2765 	(void) segkmem_create(&kvalloc);
2766 
2767 	if (kmem64_base) {
2768 		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2769 		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2770 		(void) segkmem_create(&kmem64);
2771 	}
2772 
2773 	/*
2774 	 * We're about to map out /boot.  This is the beginning of the
2775 	 * system resource management transition. We can no longer
2776 	 * call into /boot for I/O or memory allocations.
2777 	 */
2778 	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2779 	(void) segkmem_create(&kvseg);
2780 	hblk_alloc_dynamic = 1;
2781 
2782 	/*
2783 	 * we need to preallocate pages for DR operations before enabling large
2784 	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2785 	 */
2786 	memseg_remap_init();
2787 
2788 	/* at this point we are ready to use large page heap */
2789 	segkmem_heap_lp_init();
2790 
2791 	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2792 	    &kvseg32);
2793 	(void) segkmem_create(&kvseg32);
2794 
2795 	/*
2796 	 * Create a segment for the debugger.
2797 	 */
2798 	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2799 	(void) segkmem_create(&kdebugseg);
2800 
2801 	rw_exit(&kas.a_lock);
2802 }
2803 
2804 char obp_tte_str[] =
2805 	"h# %x constant MMU_PAGESHIFT "
2806 	"h# %x constant TTE8K "
2807 	"h# %x constant SFHME_SIZE "
2808 	"h# %x constant SFHME_TTE "
2809 	"h# %x constant HMEBLK_TAG "
2810 	"h# %x constant HMEBLK_NEXT "
2811 	"h# %x constant HMEBLK_MISC "
2812 	"h# %x constant HMEBLK_HME1 "
2813 	"h# %x constant NHMENTS "
2814 	"h# %x constant HBLK_SZMASK "
2815 	"h# %x constant HBLK_RANGE_SHIFT "
2816 	"h# %x constant HMEBP_HBLK "
2817 	"h# %x constant HMEBLK_ENDPA "
2818 	"h# %x constant HMEBUCKET_SIZE "
2819 	"h# %x constant HTAG_SFMMUPSZ "
2820 	"h# %x constant HTAG_BSPAGE_SHIFT "
2821 	"h# %x constant HTAG_REHASH_SHIFT "
2822 	"h# %x constant SFMMU_INVALID_SHMERID "
2823 	"h# %x constant mmu_hashcnt "
2824 	"h# %p constant uhme_hash "
2825 	"h# %p constant khme_hash "
2826 	"h# %x constant UHMEHASH_SZ "
2827 	"h# %x constant KHMEHASH_SZ "
2828 	"h# %p constant KCONTEXT "
2829 	"h# %p constant KHATID "
2830 	"h# %x constant ASI_MEM "
2831 
2832 	": PHYS-X@ ( phys -- data ) "
2833 	"   ASI_MEM spacex@ "
2834 	"; "
2835 
2836 	": PHYS-W@ ( phys -- data ) "
2837 	"   ASI_MEM spacew@ "
2838 	"; "
2839 
2840 	": PHYS-L@ ( phys -- data ) "
2841 	"   ASI_MEM spaceL@ "
2842 	"; "
2843 
2844 	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2845 	"   3 * MMU_PAGESHIFT + "
2846 	"; "
2847 
2848 	": TTE_IS_VALID ( ttep -- flag ) "
2849 	"   PHYS-X@ 0< "
2850 	"; "
2851 
2852 	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2853 	"   dup TTE8K =  if "
2854 	"      drop HBLK_RANGE_SHIFT "
2855 	"   else "
2856 	"      TTE_PAGE_SHIFT "
2857 	"   then "
2858 	"; "
2859 
2860 	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2861 	"   tuck >> swap MMU_PAGESHIFT - << "
2862 	"; "
2863 
2864 	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2865 	"   >> over xor swap                    ( hash sfmmup ) "
2866 	"   KHATID <>  if                       ( hash ) "
2867 	"      UHMEHASH_SZ and                  ( bucket ) "
2868 	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2869 	"   else                                ( hash ) "
2870 	"      KHMEHASH_SZ and                  ( bucket ) "
2871 	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2872 	"   then                                ( hmebp ) "
2873 	"; "
2874 
2875 	": HME_HASH_TABLE_SEARCH "
2876 	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2877 	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2878 	"      dup HMEBLK_ENDPA <> if     ( sfmmup hmeblkp ) ( r: hblktag ) "
2879 	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2880 	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2881 	"		  true	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2882 	"	     else						  "
2883 	"		  hmeblk_next + phys-x@ false			  "
2884 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2885 	"	     then						  "
2886 	"	  else							  "
2887 	"	     hmeblk_next + phys-x@ false			  "
2888 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2889 	"	  then							  "
2890 	"      else							  "
2891 	"         drop 0 true						  "
2892 	"      then							  "
2893 	"   until r> drop						  "
2894 	"; "
2895 
2896 	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2897 	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2898 	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2899 	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2900 	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2901 	"; "
2902 
2903 	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2904 	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2905 	"   TTE8K =  if                            ( hmeblkp addr ) "
2906 	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2907 	"   else                                   ( hmeblkp addr ) "
2908 	"      drop 0                              ( hmeblkp 0 ) "
2909 	"   then                                   ( hmeblkp hme-index ) "
2910 	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2911 	"   SFHME_TTE +                            ( ttep ) "
2912 	"; "
2913 
2914 	": unix-tte ( addr cnum -- false | tte-data true ) "
2915 	"    KCONTEXT = if                   ( addr ) "
2916 	"	KHATID                       ( addr khatid ) "
2917 	"    else                            ( addr ) "
2918 	"       drop false exit              ( false ) "
2919 	"    then "
2920 	"      ( addr khatid ) "
2921 	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2922 	"         2dup swap i HME_HASH_SHIFT  "
2923 					"( addr sfmmup sfmmup addr hmeshift ) "
2924 	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2925 	"         over i 4 pick               "
2926 				"( addr sfmmup hmebp sfmmup rehash addr ) "
2927 	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2928 	"         HME_HASH_TABLE_SEARCH       "
2929 					"( addr sfmmup { null | hmeblkp } ) "
2930 	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2931 	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2932 	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2933 	"               PHYS-X@ true          ( tte-data true ) "
2934 	"            else                     ( invalid-tte ) "
2935 	"               drop false            ( false ) "
2936 	"            then                     ( false | tte-data true ) "
2937 	"            unloop exit              ( false | tte-data true ) "
2938 	"         then                        ( addr sfmmup ) "
2939 	"      loop                           ( addr sfmmup ) "
2940 	"      2drop false                    ( false ) "
2941 	"; "
2942 ;
2943 
2944 void
2945 create_va_to_tte(void)
2946 {
2947 	char *bp;
2948 	extern int khmehash_num, uhmehash_num;
2949 	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2950 
2951 #define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2952 
2953 	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2954 
2955 	/*
2956 	 * Teach obp how to parse our sw ttes.
2957 	 */
2958 	(void) sprintf(bp, obp_tte_str,
2959 	    MMU_PAGESHIFT,
2960 	    TTE8K,
2961 	    sizeof (struct sf_hment),
2962 	    OFFSET(struct sf_hment, hme_tte),
2963 	    OFFSET(struct hme_blk, hblk_tag),
2964 	    OFFSET(struct hme_blk, hblk_nextpa),
2965 	    OFFSET(struct hme_blk, hblk_misc),
2966 	    OFFSET(struct hme_blk, hblk_hme),
2967 	    NHMENTS,
2968 	    HBLK_SZMASK,
2969 	    HBLK_RANGE_SHIFT,
2970 	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2971 	    HMEBLK_ENDPA,
2972 	    sizeof (struct hmehash_bucket),
2973 	    HTAG_SFMMUPSZ,
2974 	    HTAG_BSPAGE_SHIFT,
2975 	    HTAG_REHASH_SHIFT,
2976 	    SFMMU_INVALID_SHMERID,
2977 	    mmu_hashcnt,
2978 	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
2979 	    (caddr_t)va_to_pa((caddr_t)khme_hash),
2980 	    UHMEHASH_SZ,
2981 	    KHMEHASH_SZ,
2982 	    KCONTEXT,
2983 	    KHATID,
2984 	    ASI_MEM);
2985 	prom_interpret(bp, 0, 0, 0, 0, 0);
2986 
2987 	kobj_free(bp, MMU_PAGESIZE);
2988 }
2989 
2990 void
2991 install_va_to_tte(void)
2992 {
2993 	/*
2994 	 * advise prom that it can use unix-tte
2995 	 */
2996 	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
2997 }
2998 
2999 /*
3000  * Here we add "device-type=console" for /os-io node, for currently
3001  * our kernel console output only supports displaying text and
3002  * performing cursor-positioning operations (through kernel framebuffer
3003  * driver) and it doesn't support other functionalities required for a
3004  * standard "display" device as specified in 1275 spec. The main missing
3005  * interface defined by the 1275 spec is "draw-logo".
3006  * also see the comments above prom_stdout_is_framebuffer().
3007  */
3008 static char *create_node =
3009 	"\" /\" find-device "
3010 	"new-device "
3011 	"\" os-io\" device-name "
3012 	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
3013 	": cb-r/w  ( adr,len method$ -- #read/#written ) "
3014 	"   2>r swap 2 2r> ['] $callback  catch  if "
3015 	"      2drop 3drop 0 "
3016 	"   then "
3017 	"; "
3018 	": read ( adr,len -- #read ) "
3019 	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
3020 	"       ( retN ... ret1 N ) "
3021 	"       ?dup  if "
3022 	"               swap >r 1-  0  ?do  drop  loop  r> "
3023 	"       else "
3024 	"               -2 "
3025 	"       then "
3026 	";    "
3027 	": write ( adr,len -- #written ) "
3028 	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
3029 	"       ( retN ... ret1 N ) "
3030 	"       ?dup  if "
3031 	"               swap >r 1-  0  ?do  drop  loop  r> "
3032 	"        else "
3033 	"               0 "
3034 	"       then "
3035 	"; "
3036 	": poll-tty ( -- ) ; "
3037 	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
3038 	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
3039 	": cb-give/take ( $method -- ) "
3040 	"       0 -rot ['] $callback catch  ?dup  if "
3041 	"               >r 2drop 2drop r> throw "
3042 	"       else "
3043 	"               0  ?do  drop  loop "
3044 	"       then "
3045 	"; "
3046 	": give ( -- )  \" exit-input\" cb-give/take ; "
3047 	": take ( -- )  \" enter-input\" cb-give/take ; "
3048 	": open ( -- ok? )  true ; "
3049 	": close ( -- ) ; "
3050 	"finish-device "
3051 	"device-end ";
3052 
3053 /*
3054  * Create the OBP input/output node (FCode serial driver).
3055  * It is needed for both USB console keyboard and for
3056  * the kernel terminal emulator.  It is too early to check for a
3057  * kernel console compatible framebuffer now, so we create this
3058  * so that we're ready if we need to enable kernel terminal emulation.
3059  *
3060  * When the USB software takes over the input device at the time
3061  * consconfig runs, OBP's stdin is redirected to this node.
3062  * Whenever the FORTH user interface is used after this switch,
3063  * the node will call back into the kernel for console input.
3064  * If a serial device such as ttya or a UART with a Type 5 keyboard
3065  * attached is used, OBP takes over the serial device when the system
3066  * goes to the debugger after the system is booted.  This sharing
3067  * of the relatively simple serial device is difficult but possible.
3068  * Sharing the USB host controller is impossible due its complexity.
3069  *
3070  * Similarly to USB keyboard input redirection, after consconfig_dacf
3071  * configures a kernel console framebuffer as the standard output
3072  * device, OBP's stdout is switched to to vector through the
3073  * /os-io node into the kernel terminal emulator.
3074  */
3075 static void
3076 startup_create_io_node(void)
3077 {
3078 	prom_interpret(create_node, 0, 0, 0, 0, 0);
3079 }
3080 
3081 
3082 /*
3083  * Must be defined in platform dependent code.
3084  */
3085 extern caddr_t modtext;
3086 extern size_t modtext_sz;
3087 extern caddr_t moddata;
3088 
3089 #define	HEAPTEXT_ARENA(addr)	\
3090 	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3091 	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3092 	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3093 
3094 #define	HEAPTEXT_OVERSIZED(addr)	\
3095 	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3096 
3097 #define	HEAPTEXT_IN_NUCLEUSDATA(addr) \
3098 	(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3099 	((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3100 
3101 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3102 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3103 kmutex_t texthole_lock;
3104 
3105 char kern_bootargs[OBP_MAXPATHLEN];
3106 char kern_bootfile[OBP_MAXPATHLEN];
3107 
3108 void
3109 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3110 {
3111 	uintptr_t addr, limit;
3112 
3113 	addr = HEAPTEXT_BASE;
3114 	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3115 
3116 	/*
3117 	 * Before we initialize the text_arena, we want to punch holes in the
3118 	 * underlying heaptext_arena.  This guarantees that for any text
3119 	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3120 	 */
3121 	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3122 	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3123 		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3124 		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3125 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3126 	}
3127 
3128 	/*
3129 	 * Allocate one page at the oversize to break up the text region
3130 	 * from the oversized region.
3131 	 */
3132 	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3133 	    (void *)limit, (void *)(limit + PAGESIZE),
3134 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3135 
3136 	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3137 	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3138 	    heaptext_arena, 0, VM_SLEEP);
3139 	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3140 	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3141 }
3142 
3143 caddr_t
3144 kobj_text_alloc(vmem_t *arena, size_t size)
3145 {
3146 	caddr_t rval, better;
3147 
3148 	/*
3149 	 * First, try a sleeping allocation.
3150 	 */
3151 	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3152 
3153 	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3154 		return (rval);
3155 
3156 	/*
3157 	 * We didn't get the area that we wanted.  We're going to try to do an
3158 	 * allocation with explicit constraints.
3159 	 */
3160 	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3161 	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3162 	    VM_NOSLEEP | VM_BESTFIT);
3163 
3164 	if (better != NULL) {
3165 		/*
3166 		 * That worked.  Free our first attempt and return.
3167 		 */
3168 		vmem_free(arena, rval, size);
3169 		return (better);
3170 	}
3171 
3172 	/*
3173 	 * That didn't work; we'll have to return our first attempt.
3174 	 */
3175 	return (rval);
3176 }
3177 
3178 caddr_t
3179 kobj_texthole_alloc(caddr_t addr, size_t size)
3180 {
3181 	int arena = HEAPTEXT_ARENA(addr);
3182 	char c[30];
3183 	uintptr_t base;
3184 
3185 	if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3186 		/*
3187 		 * If this is an oversized allocation or it is allocated in
3188 		 * the nucleus data page, there is no text hole available for
3189 		 * it; return NULL.
3190 		 */
3191 		return (NULL);
3192 	}
3193 
3194 	mutex_enter(&texthole_lock);
3195 
3196 	if (texthole_arena[arena] == NULL) {
3197 		ASSERT(texthole_source[arena] == NULL);
3198 
3199 		if (arena == 0) {
3200 			texthole_source[0] = vmem_create("module_text_holesrc",
3201 			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3202 			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3203 			    0, VM_SLEEP);
3204 		} else {
3205 			base = HEAPTEXT_BASE +
3206 			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3207 
3208 			(void) snprintf(c, sizeof (c),
3209 			    "heaptext_holesrc_%d", arena);
3210 
3211 			texthole_source[arena] = vmem_create(c, (void *)base,
3212 			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3213 			    0, VM_SLEEP);
3214 		}
3215 
3216 		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3217 
3218 		texthole_arena[arena] = vmem_create(c, NULL, 0,
3219 		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3220 		    texthole_source[arena], 0, VM_SLEEP);
3221 	}
3222 
3223 	mutex_exit(&texthole_lock);
3224 
3225 	ASSERT(texthole_arena[arena] != NULL);
3226 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3227 	return (vmem_alloc(texthole_arena[arena], size,
3228 	    VM_BESTFIT | VM_NOSLEEP));
3229 }
3230 
3231 void
3232 kobj_texthole_free(caddr_t addr, size_t size)
3233 {
3234 	int arena = HEAPTEXT_ARENA(addr);
3235 
3236 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3237 	ASSERT(texthole_arena[arena] != NULL);
3238 	vmem_free(texthole_arena[arena], addr, size);
3239 }
3240 
3241 void
3242 release_bootstrap(void)
3243 {
3244 	if (&cif_init)
3245 		cif_init();
3246 }
3247