xref: /illumos-gate/usr/src/uts/sun4/os/startup.c (revision fb8f92baa78fdf1ddda6f49125fbd59366393ac8)
1  /*
2   * CDDL HEADER START
3   *
4   * The contents of this file are subject to the terms of the
5   * Common Development and Distribution License (the "License").
6   * You may not use this file except in compliance with the License.
7   *
8   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9   * or http://www.opensolaris.org/os/licensing.
10   * See the License for the specific language governing permissions
11   * and limitations under the License.
12   *
13   * When distributing Covered Code, include this CDDL HEADER in each
14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15   * If applicable, add the following below this CDDL HEADER, with the
16   * fields enclosed by brackets "[]" replaced with your own identifying
17   * information: Portions Copyright [yyyy] [name of copyright owner]
18   *
19   * CDDL HEADER END
20   */
21  
22  /*
23   * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24   * Copyright (c) 2016 by Delphix. All rights reserved.
25   * Copyright 2019 Peter Tribble.
26   */
27  
28  #include <sys/machsystm.h>
29  #include <sys/archsystm.h>
30  #include <sys/vm.h>
31  #include <sys/cpu.h>
32  #include <sys/atomic.h>
33  #include <sys/reboot.h>
34  #include <sys/kdi.h>
35  #include <sys/bootconf.h>
36  #include <sys/memlist_plat.h>
37  #include <sys/memlist_impl.h>
38  #include <sys/prom_plat.h>
39  #include <sys/prom_isa.h>
40  #include <sys/autoconf.h>
41  #include <sys/ivintr.h>
42  #include <sys/fpu/fpusystm.h>
43  #include <sys/iommutsb.h>
44  #include <vm/vm_dep.h>
45  #include <vm/seg_dev.h>
46  #include <vm/seg_kmem.h>
47  #include <vm/seg_kpm.h>
48  #include <vm/seg_map.h>
49  #include <vm/seg_kp.h>
50  #include <sys/sysconf.h>
51  #include <vm/hat_sfmmu.h>
52  #include <sys/kobj.h>
53  #include <sys/sun4asi.h>
54  #include <sys/clconf.h>
55  #include <sys/platform_module.h>
56  #include <sys/panic.h>
57  #include <sys/cpu_sgnblk_defs.h>
58  #include <sys/clock.h>
59  #include <sys/cmn_err.h>
60  #include <sys/dumphdr.h>
61  #include <sys/promif.h>
62  #include <sys/prom_debug.h>
63  #include <sys/traptrace.h>
64  #include <sys/memnode.h>
65  #include <sys/mem_cage.h>
66  #include <sys/mmu.h>
67  #include <sys/swap.h>
68  
69  extern void setup_trap_table(void);
70  extern int cpu_intrq_setup(struct cpu *);
71  extern void cpu_intrq_register(struct cpu *);
72  extern void contig_mem_init(void);
73  extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
74  extern void mach_dump_buffer_init(void);
75  extern void mach_descrip_init(void);
76  extern void mach_descrip_startup_fini(void);
77  extern void mach_memscrub(void);
78  extern void mach_fpras(void);
79  extern void mach_cpu_halt_idle(void);
80  extern void mach_hw_copy_limit(void);
81  extern void load_mach_drivers(void);
82  extern void load_tod_module(void);
83  #pragma weak load_tod_module
84  
85  extern int ndata_alloc_mmfsa(struct memlist *ndata);
86  #pragma weak ndata_alloc_mmfsa
87  
88  extern void cif_init(void);
89  #pragma weak cif_init
90  
91  extern void parse_idprom(void);
92  extern void add_vx_handler(char *, int, void (*)(cell_t *));
93  extern void mem_config_init(void);
94  extern void memseg_remap_init(void);
95  
96  extern void mach_kpm_init(void);
97  extern void pcf_init();
98  extern int size_pse_array(pgcnt_t, int);
99  extern void pg_init();
100  
101  /*
102   * External Data:
103   */
104  extern int vac_size;	/* cache size in bytes */
105  extern uint_t vac_mask;	/* VAC alignment consistency mask */
106  extern uint_t vac_colors;
107  
108  /*
109   * Global Data Definitions:
110   */
111  
112  /*
113   * XXX - Don't port this to new architectures
114   * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
115   * 'romp' has no use with a prom with an IEEE 1275 client interface.
116   * The driver doesn't use the value, but it depends on the symbol.
117   */
118  void *romp;		/* veritas driver won't load without romp 4154976 */
119  /*
120   * Declare these as initialized data so we can patch them.
121   */
122  pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
123  pgcnt_t segkpsize =
124      btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
125  uint_t segmap_percent = 6; /* Size of segmap segment */
126  
127  int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
128  int vac_copyback = 1;
129  char *cache_mode = NULL;
130  int use_mix = 1;
131  int prom_debug = 0;
132  
133  caddr_t boot_tba;		/* %tba at boot - used by kmdb */
134  uint_t	tba_taken_over = 0;
135  
136  caddr_t s_text;			/* start of kernel text segment */
137  caddr_t e_text;			/* end of kernel text segment */
138  caddr_t s_data;			/* start of kernel data segment */
139  caddr_t e_data;			/* end of kernel data segment */
140  
141  caddr_t modtext;		/* beginning of module text */
142  size_t	modtext_sz;		/* size of module text */
143  caddr_t moddata;		/* beginning of module data reserve */
144  caddr_t e_moddata;		/* end of module data reserve */
145  
146  /*
147   * End of first block of contiguous kernel in 32-bit virtual address space
148   */
149  caddr_t		econtig32;	/* end of first blk of contiguous kernel */
150  
151  caddr_t		ncbase;		/* beginning of non-cached segment */
152  caddr_t		ncend;		/* end of non-cached segment */
153  
154  size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
155  caddr_t	nalloc_base;		/* beginning of nucleus allocation */
156  caddr_t nalloc_end;		/* end of nucleus allocatable memory */
157  caddr_t valloc_base;		/* beginning of kvalloc segment	*/
158  
159  caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
160  caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
161  size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
162  caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
163  int	kmem64_szc;		/* page size code */
164  uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
165  
166  uintptr_t shm_alignment;	/* VAC address consistency modulus */
167  struct memlist *phys_install;	/* Total installed physical memory */
168  struct memlist *phys_avail;	/* Available (unreserved) physical memory */
169  struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
170  struct memlist *nopp_list;	/* pages with no backing page structs */
171  struct memlist ndata;		/* memlist of nucleus allocatable memory */
172  int memexp_flag;		/* memory expansion card flag */
173  uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
174  pgcnt_t obp_pages;		/* Physical pages used by OBP */
175  
176  /*
177   * VM data structures
178   */
179  long page_hashsz;		/* Size of page hash table (power of two) */
180  unsigned int page_hashsz_shift;	/* log2(page_hashsz) */
181  struct page *pp_base;		/* Base of system page struct array */
182  size_t pp_sz;			/* Size in bytes of page struct array */
183  struct page **page_hash;	/* Page hash table */
184  pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
185  size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
186  int pse_shift;			/* log2(pse_table_size) */
187  struct seg ktextseg;		/* Segment used for kernel executable image */
188  struct seg kvalloc;		/* Segment used for "valloc" mapping */
189  struct seg kpseg;		/* Segment used for pageable kernel virt mem */
190  struct seg ktexthole;		/* Segment used for nucleus text hole */
191  struct seg kmapseg;		/* Segment used for generic kernel mappings */
192  struct seg kpmseg;		/* Segment used for physical mapping */
193  struct seg kdebugseg;		/* Segment used for the kernel debugger */
194  
195  void *kpm_pp_base;		/* Base of system kpm_page array */
196  size_t	kpm_pp_sz;		/* Size of system kpm_page array */
197  pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
198  
199  struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
200  struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
201  struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
202  
203  int segzio_fromheap = 0;	/* zio allocations occur from heap */
204  caddr_t segzio_base;		/* Base address of segzio */
205  pgcnt_t segziosize = 0;		/* size of zio segment in pages */
206  
207  /*
208   * A static DR page_t VA map is reserved that can map the page structures
209   * for a domain's entire RA space. The pages that backs this space are
210   * dynamically allocated and need not be physically contiguous.  The DR
211   * map size is derived from KPM size.
212   */
213  int ppvm_enable = 0;		/* Static virtual map for page structs */
214  page_t *ppvm_base;		/* Base of page struct map */
215  pgcnt_t ppvm_size = 0;		/* Size of page struct map */
216  
217  /*
218   * debugger pages (if allocated)
219   */
220  struct vnode kdebugvp;
221  
222  /*
223   * VA range available to the debugger
224   */
225  const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
226  const size_t kdi_segdebugsize = SEGDEBUGSIZE;
227  
228  /*
229   * Segment for relocated kernel structures in 64-bit large RAM kernels
230   */
231  struct seg kmem64;
232  
233  struct memseg *memseg_free;
234  
235  struct vnode unused_pages_vp;
236  
237  /*
238   * VM data structures allocated early during boot.
239   */
240  size_t pagehash_sz;
241  uint64_t memlist_sz;
242  
243  char tbr_wr_addr_inited = 0;
244  
245  caddr_t	mpo_heap32_buf = NULL;
246  size_t	mpo_heap32_bufsz = 0;
247  
248  /*
249   * Static Routines:
250   */
251  static int ndata_alloc_memseg(struct memlist *, size_t);
252  static void memlist_new(uint64_t, uint64_t, struct memlist **);
253  static void memlist_add(uint64_t, uint64_t,
254  	struct memlist **, struct memlist **);
255  static void kphysm_init(void);
256  static void kvm_init(void);
257  static void install_kmem64_tte(void);
258  
259  static void startup_init(void);
260  static void startup_memlist(void);
261  static void startup_modules(void);
262  static void startup_bop_gone(void);
263  static void startup_vm(void);
264  static void startup_end(void);
265  static void setup_cage_params(void);
266  static void startup_create_io_node(void);
267  
268  static pgcnt_t npages;
269  static struct memlist *memlist;
270  void *memlist_end;
271  
272  static pgcnt_t bop_alloc_pages;
273  static caddr_t hblk_base;
274  uint_t hblk_alloc_dynamic = 0;
275  uint_t hblk1_min = H1MIN;
276  
277  
278  /*
279   * Hook for down-rev firmware
280   */
281  static void do_prom_version_check(void);
282  
283  /*
284   * After receiving a thermal interrupt, this is the number of seconds
285   * to delay before shutting off the system, assuming
286   * shutdown fails.  Use /etc/system to change the delay if this isn't
287   * large enough.
288   */
289  int thermal_powerdown_delay = 1200;
290  
291  /*
292   * Used to hold off page relocations into the cage until OBP has completed
293   * its boot-time handoff of its resources to the kernel.
294   */
295  int page_relocate_ready = 0;
296  
297  /*
298   * Indicate if kmem64 allocation was done in small chunks
299   */
300  int kmem64_smchunks = 0;
301  
302  /*
303   * Enable some debugging messages concerning memory usage...
304   */
305  #ifdef  DEBUGGING_MEM
306  static int debugging_mem;
307  static void
308  printmemlist(char *title, struct memlist *list)
309  {
310  	if (!debugging_mem)
311  		return;
312  
313  	printf("%s\n", title);
314  
315  	while (list) {
316  		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
317  		    (uint32_t)(list->ml_address >> 32),
318  		    (uint32_t)list->ml_address,
319  		    (uint32_t)(list->ml_size >> 32),
320  		    (uint32_t)(list->ml_size));
321  		list = list->ml_next;
322  	}
323  }
324  
325  void
326  printmemseg(struct memseg *memseg)
327  {
328  	if (!debugging_mem)
329  		return;
330  
331  	printf("memseg\n");
332  
333  	while (memseg) {
334  		prom_printf("\tpage = 0x%p, epage = 0x%p, "
335  		    "pfn = 0x%x, epfn = 0x%x\n",
336  		    memseg->pages, memseg->epages,
337  		    memseg->pages_base, memseg->pages_end);
338  		memseg = memseg->next;
339  	}
340  }
341  
342  #define	debug_pause(str)	halt((str))
343  #define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
344  #define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
345  #define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
346  #define	MPRINTF3(str, a, b, c) \
347  	if (debugging_mem) prom_printf((str), (a), (b), (c))
348  #else	/* DEBUGGING_MEM */
349  #define	MPRINTF(str)
350  #define	MPRINTF1(str, a)
351  #define	MPRINTF2(str, a, b)
352  #define	MPRINTF3(str, a, b, c)
353  #endif	/* DEBUGGING_MEM */
354  
355  
356  /*
357   *
358   *                    Kernel's Virtual Memory Layout.
359   *                       /-----------------------\
360   * 0xFFFFFFFF.FFFFFFFF  -|                       |-
361   *                       |   OBP's virtual page  |
362   *                       |        tables         |
363   * 0xFFFFFFFC.00000000  -|-----------------------|-
364   *                       :                       :
365   *                       :                       :
366   *                      -|-----------------------|-
367   *                       |       segzio          | (base and size vary)
368   * 0xFFFFFE00.00000000  -|-----------------------|-
369   *                       |                       |  Ultrasparc I/II support
370   *                       |    segkpm segment     |  up to 2TB of physical
371   *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
372   *                       |                       |
373   * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
374   *                       :                       :
375   *                       :                       :
376   * 0xFFFFF810.00000000  -|-----------------------|- hole_end
377   *                       |                       |      ^
378   *                       |  UltraSPARC I/II call |      |
379   *                       | bug requires an extra |      |
380   *                       | 4 GB of space between |      |
381   *                       |   hole and used RAM   |	|
382   *                       |                       |      |
383   * 0xFFFFF800.00000000  -|-----------------------|-     |
384   *                       |                       |      |
385   *                       | Virtual Address Hole  |   UltraSPARC
386   *                       |  on UltraSPARC I/II   |  I/II * ONLY *
387   *                       |                       |      |
388   * 0x00000800.00000000  -|-----------------------|-     |
389   *                       |                       |      |
390   *                       |  UltraSPARC I/II call |      |
391   *                       | bug requires an extra |      |
392   *                       | 4 GB of space between |      |
393   *                       |   hole and used RAM   |      |
394   *                       |                       |      v
395   * 0x000007FF.00000000  -|-----------------------|- hole_start -----
396   *                       :                       :		   ^
397   *                       :                       :		   |
398   *                       |-----------------------|                 |
399   *                       |                       |                 |
400   *                       |  ecache flush area    |                 |
401   *                       |  (twice largest e$)   |                 |
402   *                       |                       |                 |
403   * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
404   *                       | overmapped area       |   alignend_end  |
405   *                       | (kmem64_alignsize     |		   |
406   *                       |  boundary)            |		   |
407   * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
408   *                       |                       |		   |
409   *                       |   64-bit kernel ONLY  |		   |
410   *                       |                       |		   |
411   *                       |    kmem64 segment     |		   |
412   *                       |                       |		   |
413   *                       | (Relocated extra HME  |	     Approximately
414   *                       |   block allocations,  |	    1 TB of virtual
415   *                       |   memnode freelists,  |	     address space
416   *                       |    HME hash buckets,  |		   |
417   *                       | mml_table, kpmp_table,|		   |
418   *                       |  page_t array and     |		   |
419   *                       |  hashblock pool to    |		   |
420   *                       |   avoid hard-coded    |		   |
421   *                       |     32-bit vaddr      |		   |
422   *                       |     limitations)      |		   |
423   *                       |                       |		   v
424   * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
425   *                       |                       |
426   *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
427   *                       |                       |
428   * 0x00000300.00000000  -|-----------------------|- SYSBASE
429   *                       :                       :
430   *                       :                       :
431   *                      -|-----------------------|-
432   *                       |                       |
433   *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
434   *                       |                       |               256G MAX)
435   * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
436   *                       :                       :
437   *                       :                       :
438   *                      -|-----------------------|-
439   *                       |                       |
440   *                       |       segkp           |    SEGKPSIZE (2GB)
441   *                       |                       |
442   *                       |                       |
443   * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
444   *                       |                       |
445   * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
446   *                       |                       |       (SEGKPBASE - 0x400000)
447   * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
448   *                       |                       |       (MEMSCRUBBASE - NCARGS)
449   * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
450   *                       |                       |       (ARGSBASE - PPMAPSIZE)
451   * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
452   *                       |                       |
453   * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
454   *                       |                       |
455   * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
456   *                       :                       :
457   *                       :                       :
458   * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
459   *                       |                       |
460   *                       |         OBP           |
461   *                       |                       |
462   * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
463   *                       |         kmdb          |
464   * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
465   *                       :                       :
466   *                       :                       :
467   * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
468   *                       |                       |
469   *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
470   *                       |                       |    ~64MB)
471   *			-|-----------------------|
472   *			 |	IVSIZE		 |
473   * 0x00000000.70004000  -|-----------------------|
474   *                       |     panicbuf          |
475   * 0x00000000.70002000	-|-----------------------|
476   *			 |	PAGESIZE	 |
477   * 0x00000000.70000000  -|-----------------------|- SYSBASE32
478   *                       |       boot-time       |
479   *                       |    temporary space    |
480   * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
481   *                       :                       :
482   *                       :                       :
483   *                       |                       |
484   *                       |-----------------------|- econtig32
485   *                       |    vm structures      |
486   * 0x00000000.01C00000   |-----------------------|- nalloc_end
487   *                       |         TSBs          |
488   *                       |-----------------------|- end/nalloc_base
489   *                       |   kernel data & bss   |
490   * 0x00000000.01800000  -|-----------------------|
491   *                       :   nucleus text hole   :
492   * 0x00000000.01400000  -|-----------------------|
493   *                       :                       :
494   *                       |-----------------------|
495   *                       |      module text      |
496   *                       |-----------------------|- e_text/modtext
497   *                       |      kernel text      |
498   *                       |-----------------------|
499   *                       |    trap table (48k)   |
500   * 0x00000000.01000000  -|-----------------------|- KERNELBASE
501   *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
502   *                       |-----------------------|
503   *                       |                       |
504   *                       |        invalid        |
505   *                       |                       |
506   * 0x00000000.00000000  _|_______________________|
507   *
508   *
509   *
510   *                   32-bit User Virtual Memory Layout.
511   *                       /-----------------------\
512   *                       |                       |
513   *                       |        invalid        |
514   *                       |                       |
515   *          0xFFC00000  -|-----------------------|- USERLIMIT
516   *                       |       user stack      |
517   *                       :                       :
518   *                       :                       :
519   *                       :                       :
520   *                       |       user data       |
521   *                      -|-----------------------|-
522   *                       |       user text       |
523   *          0x00002000  -|-----------------------|-
524   *                       |       invalid         |
525   *          0x00000000  _|_______________________|
526   *
527   *
528   *
529   *                   64-bit User Virtual Memory Layout.
530   *                       /-----------------------\
531   *                       |                       |
532   *                       |        invalid        |
533   *                       |                       |
534   *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
535   *                       |       user stack      |
536   *                       :                       :
537   *                       :                       :
538   *                       :                       :
539   *                       |       user data       |
540   *                      -|-----------------------|-
541   *                       |       user text       |
542   *  0x00000000.01000000 -|-----------------------|-
543   *                       |       invalid         |
544   *  0x00000000.00000000 _|_______________________|
545   */
546  
547  extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
548  extern uint64_t ecache_flush_address(void);
549  
550  #pragma weak load_platform_modules
551  #pragma weak plat_startup_memlist
552  #pragma weak ecache_init_scrub_flush_area
553  #pragma weak ecache_flush_address
554  
555  
556  /*
557   * By default the DR Cage is enabled for maximum OS
558   * MPSS performance.  Users needing to disable the cage mechanism
559   * can set this variable to zero via /etc/system.
560   * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
561   * will result in loss of DR functionality.
562   * Platforms wishing to disable kernel Cage by default
563   * should do so in their set_platform_defaults() routine.
564   */
565  int	kernel_cage_enable = 1;
566  
567  static void
568  setup_cage_params(void)
569  {
570  	void (*func)(void);
571  
572  	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
573  	if (func != NULL) {
574  		(*func)();
575  		return;
576  	}
577  
578  	if (kernel_cage_enable == 0) {
579  		return;
580  	}
581  	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
582  
583  	if (kcage_on) {
584  		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
585  	} else {
586  		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
587  	}
588  
589  }
590  
591  /*
592   * Machine-dependent startup code
593   */
594  void
595  startup(void)
596  {
597  	startup_init();
598  	if (&startup_platform)
599  		startup_platform();
600  	startup_memlist();
601  	startup_modules();
602  	setup_cage_params();
603  	startup_bop_gone();
604  	startup_vm();
605  	startup_end();
606  }
607  
608  struct regs sync_reg_buf;
609  uint64_t sync_tt;
610  
611  void
612  sync_handler(void)
613  {
614  	struct  panic_trap_info	ti;
615  	int i;
616  
617  	/*
618  	 * Prevent trying to talk to the other CPUs since they are
619  	 * sitting in the prom and won't reply.
620  	 */
621  	for (i = 0; i < NCPU; i++) {
622  		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
623  			cpu[i]->cpu_flags &= ~CPU_READY;
624  			cpu[i]->cpu_flags |= CPU_QUIESCED;
625  			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
626  		}
627  	}
628  
629  	/*
630  	 * Force a serial dump, since there are no CPUs to help.
631  	 */
632  	dump_plat_mincpu = 0;
633  
634  	/*
635  	 * We've managed to get here without going through the
636  	 * normal panic code path. Try and save some useful
637  	 * information.
638  	 */
639  	if (!panicstr && (curthread->t_panic_trap == NULL)) {
640  		ti.trap_type = sync_tt;
641  		ti.trap_regs = &sync_reg_buf;
642  		ti.trap_addr = NULL;
643  		ti.trap_mmu_fsr = 0x0;
644  
645  		curthread->t_panic_trap = &ti;
646  	}
647  
648  	/*
649  	 * If we're re-entering the panic path, update the signature
650  	 * block so that the SC knows we're in the second part of panic.
651  	 */
652  	if (panicstr)
653  		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
654  
655  	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
656  	panic("sync initiated");
657  }
658  
659  
660  static void
661  startup_init(void)
662  {
663  	/*
664  	 * We want to save the registers while we're still in OBP
665  	 * so that we know they haven't been fiddled with since.
666  	 * (In principle, OBP can't change them just because it
667  	 * makes a callback, but we'd rather not depend on that
668  	 * behavior.)
669  	 */
670  	char		sync_str[] =
671  	    "warning @ warning off : sync "
672  	    "%%tl-c %%tstate h# %p x! "
673  	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
674  	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
675  	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
676  	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
677  	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
678  	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
679  	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
680  	    "sync ; warning !";
681  
682  	/*
683  	 * 20 == num of %p substrings
684  	 * 16 == max num of chars %p will expand to.
685  	 */
686  	char		bp[sizeof (sync_str) + 16 * 20];
687  
688  	/*
689  	 * Initialize ptl1 stack for the 1st CPU.
690  	 */
691  	ptl1_init_cpu(&cpu0);
692  
693  	/*
694  	 * Initialize the address map for cache consistent mappings
695  	 * to random pages; must be done after vac_size is set.
696  	 */
697  	ppmapinit();
698  
699  	/*
700  	 * Initialize the PROM callback handler.
701  	 */
702  	init_vx_handler();
703  
704  	/*
705  	 * have prom call sync_callback() to handle the sync and
706  	 * save some useful information which will be stored in the
707  	 * core file later.
708  	 */
709  	(void) sprintf((char *)bp, sync_str,
710  	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
711  	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
712  	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
713  	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
714  	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
715  	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
716  	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
717  	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
718  	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
719  	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
720  	prom_interpret(bp, 0, 0, 0, 0, 0);
721  	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
722  }
723  
724  
725  size_t
726  calc_pp_sz(pgcnt_t npages)
727  {
728  
729  	return (npages * sizeof (struct page));
730  }
731  
732  size_t
733  calc_kpmpp_sz(pgcnt_t npages)
734  {
735  
736  	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
737  	kpm_pgsz = 1ull << kpm_pgshft;
738  	kpm_pgoff = kpm_pgsz - 1;
739  	kpmp2pshft = kpm_pgshft - PAGESHIFT;
740  	kpmpnpgs = 1 << kpmp2pshft;
741  
742  	if (kpm_smallpages == 0) {
743  		/*
744  		 * Avoid fragmentation problems in kphysm_init()
745  		 * by allocating for all of physical memory
746  		 */
747  		kpm_npages = ptokpmpr(physinstalled);
748  		return (kpm_npages * sizeof (kpm_page_t));
749  	} else {
750  		kpm_npages = npages;
751  		return (kpm_npages * sizeof (kpm_spage_t));
752  	}
753  }
754  
755  size_t
756  calc_pagehash_sz(pgcnt_t npages)
757  {
758  	/* LINTED */
759  	ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
760  	/*
761  	 * The page structure hash table size is a power of 2
762  	 * such that the average hash chain length is PAGE_HASHAVELEN.
763  	 */
764  	page_hashsz = npages / PAGE_HASHAVELEN;
765  	page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
766  	    highbit(page_hashsz));
767  	page_hashsz = 1 << page_hashsz_shift;
768  	return (page_hashsz * sizeof (struct page *));
769  }
770  
771  int testkmem64_smchunks = 0;
772  
773  int
774  alloc_kmem64(caddr_t base, caddr_t end)
775  {
776  	int i;
777  	caddr_t aligned_end = NULL;
778  
779  	if (testkmem64_smchunks)
780  		return (1);
781  
782  	/*
783  	 * Make one large memory alloc after figuring out the 64-bit size. This
784  	 * will enable use of the largest page size appropriate for the system
785  	 * architecture.
786  	 */
787  	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
788  	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
789  	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
790  		size_t alloc_size, alignsize;
791  #if !defined(C_OBP)
792  		unsigned long long pa;
793  #endif	/* !C_OBP */
794  
795  		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
796  			continue;
797  		alignsize = TTEBYTES(i);
798  		kmem64_szc = i;
799  
800  		/* limit page size for small memory */
801  		if (mmu_btop(alignsize) > (npages >> 2))
802  			continue;
803  
804  		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
805  		alloc_size = aligned_end - base;
806  #if !defined(C_OBP)
807  		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
808  			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
809  				kmem64_pabase = pa;
810  				kmem64_aligned_end = aligned_end;
811  				install_kmem64_tte();
812  				break;
813  			} else {
814  				prom_free_phys(alloc_size, pa);
815  			}
816  		}
817  #else	/* !C_OBP */
818  		if (prom_alloc(base, alloc_size, alignsize) == base) {
819  			kmem64_pabase = va_to_pa(kmem64_base);
820  			kmem64_aligned_end = aligned_end;
821  			break;
822  		}
823  #endif	/* !C_OBP */
824  		if (i == TTE8K) {
825  #ifdef sun4v
826  			/* return failure to try small allocations */
827  			return (1);
828  #else
829  			prom_panic("kmem64 allocation failure");
830  #endif
831  		}
832  	}
833  	ASSERT(aligned_end != NULL);
834  	return (0);
835  }
836  
837  static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
838  static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
839  
840  #if !defined(C_OBP)
841  /*
842   * Install a temporary tte handler in OBP for kmem64 area.
843   *
844   * We map kmem64 area with large pages before the trap table is taken
845   * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
846   * the same area. Duplicate tlb entries with different page sizes
847   * cause unpredicatble behavior.  To avoid this, we don't create
848   * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
849   * OBP).  Instead, we manage translations with a temporary va>tte-data
850   * handler (kmem64-tte).  This handler is replaced by unix-tte when
851   * the trap table is taken over.
852   *
853   * The temporary handler knows the physical address of the kmem64
854   * area. It uses the prom's pgmap@ Forth word for other addresses.
855   *
856   * We have to use BOP_ALLOC() method for C-OBP platforms because
857   * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
858   * sun4u platforms. On sun4u we flush tlb after trap table is taken
859   * over if we use large pages for kernel heap and kmem64. Since sun4u
860   * prom (unlike sun4v) calls va>tte-data first for client address
861   * translation prom's ttes for kmem64 can't get into TLB even if we
862   * later switch to prom's trap table again. C-OBP uses 4M pages for
863   * client mappings when possible so on all platforms we get the
864   * benefit from large mappings for kmem64 area immediately during
865   * boot.
866   *
867   * pseudo code:
868   * if (context != 0) {
869   *	return false
870   * } else if (miss_va in range[kmem64_base, kmem64_end)) {
871   *	tte = tte_template +
872   *		(((miss_va & pagemask) - kmem64_base));
873   *	return tte, true
874   * } else {
875   *	return pgmap@ result
876   * }
877   */
878  char kmem64_obp_str[] =
879  	"h# %lx constant kmem64-base "
880  	"h# %lx constant kmem64-end "
881  	"h# %lx constant kmem64-pagemask "
882  	"h# %lx constant kmem64-template "
883  
884  	": kmem64-tte ( addr cnum -- false | tte-data true ) "
885  	"    if                                       ( addr ) "
886  	"       drop false exit then                  ( false ) "
887  	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
888  	"	kmem64-pagemask and                   ( addr' ) "
889  	"	kmem64-base -                         ( addr' ) "
890  	"	kmem64-template +                     ( tte ) "
891  	"	true                                  ( tte true ) "
892  	"    else                                     ( addr ) "
893  	"	pgmap@                                ( tte ) "
894  	"       dup 0< if true else drop false then   ( tte true  |  false ) "
895  	"    then                                     ( tte true  |  false ) "
896  	"; "
897  
898  	"' kmem64-tte is va>tte-data "
899  ;
900  
901  static void
902  install_kmem64_tte()
903  {
904  	char b[sizeof (kmem64_obp_str) + (4 * 16)];
905  	tte_t tte;
906  
907  	PRM_DEBUG(kmem64_pabase);
908  	PRM_DEBUG(kmem64_szc);
909  	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
910  	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
911  	PRM_DEBUG(tte.ll);
912  	(void) sprintf(b, kmem64_obp_str,
913  	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
914  	ASSERT(strlen(b) < sizeof (b));
915  	prom_interpret(b, 0, 0, 0, 0, 0);
916  }
917  #endif	/* !C_OBP */
918  
919  /*
920   * As OBP takes up some RAM when the system boots, pages will already be "lost"
921   * to the system and reflected in npages by the time we see it.
922   *
923   * We only want to allocate kernel structures in the 64-bit virtual address
924   * space on systems with enough RAM to make the overhead of keeping track of
925   * an extra kernel memory segment worthwhile.
926   *
927   * Since OBP has already performed its memory allocations by this point, if we
928   * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
929   * memory in the 64-bit virtual address space; otherwise keep allocations
930   * contiguous with we've mapped so far in the 32-bit virtual address space.
931   */
932  #define	MINMOVE_RAM_MB	((size_t)1900)
933  #define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
934  #define	BYTES_TO_MB(b) ((b) / 1048576ul)
935  
936  pgcnt_t	tune_npages = (pgcnt_t)
937  	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
938  
939  #pragma weak page_set_colorequiv_arr_cpu
940  extern void page_set_colorequiv_arr_cpu(void);
941  extern void page_set_colorequiv_arr(void);
942  
943  static pgcnt_t ramdisk_npages;
944  static struct memlist *old_phys_avail;
945  
946  kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
947  
948  static void
949  startup_memlist(void)
950  {
951  	size_t hmehash_sz, pagelist_sz, tt_sz;
952  	size_t psetable_sz;
953  	caddr_t alloc_base;
954  	caddr_t memspace;
955  	struct memlist *cur;
956  	size_t syslimit = (size_t)SYSLIMIT;
957  	size_t sysbase = (size_t)SYSBASE;
958  
959  	/*
960  	 * Initialize enough of the system to allow kmem_alloc to work by
961  	 * calling boot to allocate its memory until the time that
962  	 * kvm_init is completed.  The page structs are allocated after
963  	 * rounding up end to the nearest page boundary; the memsegs are
964  	 * initialized and the space they use comes from the kernel heap.
965  	 * With appropriate initialization, they can be reallocated later
966  	 * to a size appropriate for the machine's configuration.
967  	 *
968  	 * At this point, memory is allocated for things that will never
969  	 * need to be freed, this used to be "valloced".  This allows a
970  	 * savings as the pages don't need page structures to describe
971  	 * them because them will not be managed by the vm system.
972  	 */
973  
974  	/*
975  	 * We're loaded by boot with the following configuration (as
976  	 * specified in the sun4u/conf/Mapfile):
977  	 *
978  	 *	text:		4 MB chunk aligned on a 4MB boundary
979  	 *	data & bss:	4 MB chunk aligned on a 4MB boundary
980  	 *
981  	 * These two chunks will eventually be mapped by 2 locked 4MB
982  	 * ttes and will represent the nucleus of the kernel.  This gives
983  	 * us some free space that is already allocated, some or all of
984  	 * which is made available to kernel module text.
985  	 *
986  	 * The free space in the data-bss chunk is used for nucleus
987  	 * allocatable data structures and we reserve it using the
988  	 * nalloc_base and nalloc_end variables.  This space is currently
989  	 * being used for hat data structures required for tlb miss
990  	 * handling operations.  We align nalloc_base to a l2 cache
991  	 * linesize because this is the line size the hardware uses to
992  	 * maintain cache coherency.
993  	 * 512K is carved out for module data.
994  	 */
995  
996  	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
997  	e_moddata = moddata + MODDATA;
998  	nalloc_base = e_moddata;
999  
1000  	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
1001  	valloc_base = nalloc_base;
1002  
1003  	/*
1004  	 * Calculate the start of the data segment.
1005  	 */
1006  	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1007  		prom_panic("nucleus data overflow");
1008  
1009  	PRM_DEBUG(moddata);
1010  	PRM_DEBUG(nalloc_base);
1011  	PRM_DEBUG(nalloc_end);
1012  
1013  	/*
1014  	 * Remember any slop after e_text so we can give it to the modules.
1015  	 */
1016  	PRM_DEBUG(e_text);
1017  	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1018  	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1019  		prom_panic("nucleus text overflow");
1020  	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1021  	    modtext;
1022  	PRM_DEBUG(modtext);
1023  	PRM_DEBUG(modtext_sz);
1024  
1025  	init_boot_memlists();
1026  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1027  	    &boot_physavail, &boot_physavail_len,
1028  	    &boot_virtavail, &boot_virtavail_len);
1029  
1030  	/*
1031  	 * Remember what the physically available highest page is
1032  	 * so that dumpsys works properly, and find out how much
1033  	 * memory is installed.
1034  	 */
1035  	installed_top_size_memlist_array(boot_physinstalled,
1036  	    boot_physinstalled_len, &physmax, &physinstalled);
1037  	PRM_DEBUG(physinstalled);
1038  	PRM_DEBUG(physmax);
1039  
1040  	/* Fill out memory nodes config structure */
1041  	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1042  
1043  	/*
1044  	 * npages is the maximum of available physical memory possible.
1045  	 * (ie. it will never be more than this)
1046  	 *
1047  	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1048  	 * using phys_avail will underestimate what will end up being freed.
1049  	 * A better initial guess is just total memory minus the kernel text
1050  	 */
1051  	npages = physinstalled - btop(MMU_PAGESIZE4M);
1052  
1053  	/*
1054  	 * First allocate things that can go in the nucleus data page
1055  	 * (fault status, TSBs, dmv, CPUs)
1056  	 */
1057  	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1058  
1059  	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1060  		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1061  
1062  	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1063  		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1064  
1065  	if (ndata_alloc_dmv(&ndata) != 0)
1066  		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1067  
1068  	if (ndata_alloc_page_mutexs(&ndata) != 0)
1069  		cmn_err(CE_PANIC,
1070  		    "no more nucleus memory after page free lists alloc");
1071  
1072  	if (ndata_alloc_hat(&ndata) != 0)
1073  		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1074  
1075  	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1076  		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1077  
1078  	/*
1079  	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1080  	 *
1081  	 * There are comments all over the SFMMU code warning of dire
1082  	 * consequences if the TSBs are moved out of 32-bit space.  This
1083  	 * is largely because the asm code uses "sethi %hi(addr)"-type
1084  	 * instructions which will not provide the expected result if the
1085  	 * address is a 64-bit one.
1086  	 *
1087  	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1088  	 */
1089  	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1090  	PRM_DEBUG(alloc_base);
1091  
1092  	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1093  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1094  	PRM_DEBUG(alloc_base);
1095  
1096  	/*
1097  	 * Allocate IOMMU TSB array.  We do this here so that the physical
1098  	 * memory gets deducted from the PROM's physical memory list.
1099  	 */
1100  	alloc_base = iommu_tsb_init(alloc_base);
1101  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1102  	PRM_DEBUG(alloc_base);
1103  
1104  	/*
1105  	 * Allow for an early allocation of physically contiguous memory.
1106  	 */
1107  	alloc_base = contig_mem_prealloc(alloc_base, npages);
1108  
1109  	/*
1110  	 * Platforms like Starcat and OPL need special structures assigned in
1111  	 * 32-bit virtual address space because their probing routines execute
1112  	 * FCode, and FCode can't handle 64-bit virtual addresses...
1113  	 */
1114  	if (&plat_startup_memlist) {
1115  		alloc_base = plat_startup_memlist(alloc_base);
1116  		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1117  		    ecache_alignsize);
1118  		PRM_DEBUG(alloc_base);
1119  	}
1120  
1121  	/*
1122  	 * Save off where the contiguous allocations to date have ended
1123  	 * in econtig32.
1124  	 */
1125  	econtig32 = alloc_base;
1126  	PRM_DEBUG(econtig32);
1127  	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1128  		cmn_err(CE_PANIC, "econtig32 too big");
1129  
1130  	pp_sz = calc_pp_sz(npages);
1131  	PRM_DEBUG(pp_sz);
1132  	if (kpm_enable) {
1133  		kpm_pp_sz = calc_kpmpp_sz(npages);
1134  		PRM_DEBUG(kpm_pp_sz);
1135  	}
1136  
1137  	hmehash_sz = calc_hmehash_sz(npages);
1138  	PRM_DEBUG(hmehash_sz);
1139  
1140  	pagehash_sz = calc_pagehash_sz(npages);
1141  	PRM_DEBUG(pagehash_sz);
1142  
1143  	pagelist_sz = calc_free_pagelist_sz();
1144  	PRM_DEBUG(pagelist_sz);
1145  
1146  #ifdef	TRAPTRACE
1147  	tt_sz = calc_traptrace_sz();
1148  	PRM_DEBUG(tt_sz);
1149  #else
1150  	tt_sz = 0;
1151  #endif	/* TRAPTRACE */
1152  
1153  	/*
1154  	 * Place the array that protects pp->p_selock in the kmem64 wad.
1155  	 */
1156  	pse_shift = size_pse_array(npages, max_ncpus);
1157  	PRM_DEBUG(pse_shift);
1158  	pse_table_size = 1 << pse_shift;
1159  	PRM_DEBUG(pse_table_size);
1160  	psetable_sz = roundup(
1161  	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1162  	PRM_DEBUG(psetable_sz);
1163  
1164  	/*
1165  	 * Now allocate the whole wad
1166  	 */
1167  	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1168  	    pagelist_sz + tt_sz + psetable_sz;
1169  	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1170  	kmem64_base = (caddr_t)syslimit;
1171  	kmem64_end = kmem64_base + kmem64_sz;
1172  	if (alloc_kmem64(kmem64_base, kmem64_end)) {
1173  		/*
1174  		 * Attempt for kmem64 to allocate one big
1175  		 * contiguous chunk of memory failed.
1176  		 * We get here because we are sun4v.
1177  		 * We will proceed by breaking up
1178  		 * the allocation into two attempts.
1179  		 * First, we allocate kpm_pp_sz, hmehash_sz,
1180  		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1181  		 * one contiguous chunk. This is a much smaller
1182  		 * chunk and we should get it, if not we panic.
1183  		 * Note that hmehash and tt need to be physically
1184  		 * (in the real address sense) contiguous.
1185  		 * Next, we use bop_alloc_chunk() to
1186  		 * to allocate the page_t structures.
1187  		 * This will allow the page_t to be allocated
1188  		 * in multiple smaller chunks.
1189  		 * In doing so, the assumption that page_t is
1190  		 * physically contiguous no longer hold, this is ok
1191  		 * for sun4v but not for sun4u.
1192  		 */
1193  		size_t  tmp_size;
1194  		caddr_t tmp_base;
1195  
1196  		pp_sz  = roundup(pp_sz, PAGESIZE);
1197  
1198  		/*
1199  		 * Allocate kpm_pp_sz, hmehash_sz,
1200  		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1201  		 */
1202  		tmp_base = kmem64_base + pp_sz;
1203  		tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1204  		    pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1205  		if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1206  			prom_panic("kmem64 prom_alloc contig failed");
1207  		PRM_DEBUG(tmp_base);
1208  		PRM_DEBUG(tmp_size);
1209  
1210  		/*
1211  		 * Allocate the page_ts
1212  		 */
1213  		if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1214  			prom_panic("kmem64 bop_alloc_chunk page_t failed");
1215  		PRM_DEBUG(kmem64_base);
1216  		PRM_DEBUG(pp_sz);
1217  
1218  		kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1219  		ASSERT(kmem64_aligned_end >= kmem64_end);
1220  
1221  		kmem64_smchunks = 1;
1222  	} else {
1223  
1224  		/*
1225  		 * We need to adjust pp_sz for the normal
1226  		 * case where kmem64 can allocate one large chunk
1227  		 */
1228  		if (kpm_smallpages == 0) {
1229  			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1230  		} else {
1231  			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1232  			    sizeof (kpm_spage_t));
1233  		}
1234  		pp_sz = npages * sizeof (struct page);
1235  	}
1236  
1237  	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1238  		cmn_err(CE_PANIC, "not enough kmem64 space");
1239  	PRM_DEBUG(kmem64_base);
1240  	PRM_DEBUG(kmem64_end);
1241  	PRM_DEBUG(kmem64_aligned_end);
1242  
1243  	/*
1244  	 * ... and divy it up
1245  	 */
1246  	alloc_base = kmem64_base;
1247  
1248  	pp_base = (page_t *)alloc_base;
1249  	alloc_base += pp_sz;
1250  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1251  	PRM_DEBUG(pp_base);
1252  	PRM_DEBUG(npages);
1253  
1254  	if (kpm_enable) {
1255  		kpm_pp_base = alloc_base;
1256  		if (kpm_smallpages == 0) {
1257  			/* kpm_npages based on physinstalled, don't reset */
1258  			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1259  		} else {
1260  			kpm_npages = ptokpmpr(npages);
1261  			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1262  		}
1263  		alloc_base += kpm_pp_sz;
1264  		alloc_base =
1265  		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1266  		PRM_DEBUG(kpm_pp_base);
1267  	}
1268  
1269  	alloc_base = alloc_hmehash(alloc_base);
1270  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1271  	PRM_DEBUG(alloc_base);
1272  
1273  	page_hash = (page_t **)alloc_base;
1274  	alloc_base += pagehash_sz;
1275  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1276  	PRM_DEBUG(page_hash);
1277  
1278  	alloc_base = alloc_page_freelists(alloc_base);
1279  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1280  	PRM_DEBUG(alloc_base);
1281  
1282  #ifdef	TRAPTRACE
1283  	ttrace_buf = alloc_base;
1284  	alloc_base += tt_sz;
1285  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1286  	PRM_DEBUG(alloc_base);
1287  #endif	/* TRAPTRACE */
1288  
1289  	pse_mutex = (pad_mutex_t *)alloc_base;
1290  	alloc_base += psetable_sz;
1291  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1292  	PRM_DEBUG(alloc_base);
1293  
1294  	/*
1295  	 * Note that if we use small chunk allocations for
1296  	 * kmem64, we need to ensure kmem64_end is the same as
1297  	 * kmem64_aligned_end to prevent subsequent logic from
1298  	 * trying to reuse the overmapping.
1299  	 * Otherwise we adjust kmem64_end to what we really allocated.
1300  	 */
1301  	if (kmem64_smchunks) {
1302  		kmem64_end = kmem64_aligned_end;
1303  	} else {
1304  		kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1305  	}
1306  	kmem64_sz = kmem64_end - kmem64_base;
1307  
1308  	if (&ecache_init_scrub_flush_area) {
1309  		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1310  		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1311  	}
1312  
1313  	/*
1314  	 * If physmem is patched to be non-zero, use it instead of
1315  	 * the monitor value unless physmem is larger than the total
1316  	 * amount of memory on hand.
1317  	 */
1318  	if (physmem == 0 || physmem > npages)
1319  		physmem = npages;
1320  
1321  	/*
1322  	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1323  	 * is mounted as root. This memory is held down by OBP and unlike
1324  	 * the stub boot_archive is never released.
1325  	 *
1326  	 * In order to get things sized correctly on lower memory
1327  	 * machines (where the memory used by the ramdisk represents
1328  	 * a significant portion of memory), physmem is adjusted.
1329  	 *
1330  	 * This is done by subtracting the ramdisk_size which is set
1331  	 * to the size of the ramdisk (in Kb) in /etc/system at the
1332  	 * time the miniroot archive is constructed.
1333  	 */
1334  	if (root_is_ramdisk == B_TRUE) {
1335  		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1336  		physmem -= ramdisk_npages;
1337  	}
1338  
1339  	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1340  		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1341  
1342  	/*
1343  	 * Allocate space for the interrupt vector table.
1344  	 */
1345  	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1346  	if (memspace != (caddr_t)intr_vec_table)
1347  		prom_panic("interrupt vector table allocation failure");
1348  
1349  	/*
1350  	 * Between now and when we finish copying in the memory lists,
1351  	 * allocations happen so the space gets fragmented and the
1352  	 * lists longer.  Leave enough space for lists twice as
1353  	 * long as we have now; then roundup to a pagesize.
1354  	 */
1355  	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1356  	    prom_phys_avail_len() + prom_virt_avail_len());
1357  	memlist_sz *= 2;
1358  	memlist_sz = roundup(memlist_sz, PAGESIZE);
1359  	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1360  	if (memspace == NULL)
1361  		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1362  
1363  	memlist = (struct memlist *)memspace;
1364  	memlist_end = (char *)memspace + memlist_sz;
1365  	PRM_DEBUG(memlist);
1366  	PRM_DEBUG(memlist_end);
1367  
1368  	PRM_DEBUG(sysbase);
1369  	PRM_DEBUG(syslimit);
1370  	kernelheap_init((void *)sysbase, (void *)syslimit,
1371  	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1372  
1373  	/*
1374  	 * Take the most current snapshot we can by calling mem-update.
1375  	 */
1376  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1377  	    &boot_physavail, &boot_physavail_len,
1378  	    &boot_virtavail, &boot_virtavail_len);
1379  
1380  	/*
1381  	 * Remove the space used by prom_alloc from the kernel heap
1382  	 * plus the area actually used by the OBP (if any)
1383  	 * ignoring virtual addresses in virt_avail, above syslimit.
1384  	 */
1385  	virt_avail = memlist;
1386  	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1387  
1388  	for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1389  		uint64_t range_base, range_size;
1390  
1391  		if ((range_base = cur->ml_address + cur->ml_size) <
1392  		    (uint64_t)sysbase)
1393  			continue;
1394  		if (range_base >= (uint64_t)syslimit)
1395  			break;
1396  		/*
1397  		 * Limit the range to end at syslimit.
1398  		 */
1399  		range_size = MIN(cur->ml_next->ml_address,
1400  		    (uint64_t)syslimit) - range_base;
1401  		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1402  		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1403  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1404  	}
1405  
1406  	phys_avail = memlist;
1407  	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1408  
1409  	/*
1410  	 * Add any extra memory at the end of the ndata region if there's at
1411  	 * least a page to add.  There might be a few more pages available in
1412  	 * the middle of the ndata region, but for now they are ignored.
1413  	 */
1414  	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1415  	if (nalloc_base == NULL)
1416  		nalloc_base = nalloc_end;
1417  	ndata_remain_sz = nalloc_end - nalloc_base;
1418  
1419  	/*
1420  	 * Copy physinstalled list into kernel space.
1421  	 */
1422  	phys_install = memlist;
1423  	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1424  
1425  	/*
1426  	 * Create list of physical addrs we don't need pp's for:
1427  	 * kernel text 4M page
1428  	 * kernel data 4M page - ndata_remain_sz
1429  	 * kmem64 pages
1430  	 *
1431  	 * NB if adding any pages here, make sure no kpm page
1432  	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1433  	 */
1434  	nopp_list = memlist;
1435  	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1436  	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1437  	    &memlist, &nopp_list);
1438  
1439  	/* Don't add to nopp_list if kmem64 was allocated in smchunks */
1440  	if (!kmem64_smchunks)
1441  		memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1442  
1443  	if ((caddr_t)memlist > (memspace + memlist_sz))
1444  		prom_panic("memlist overflow");
1445  
1446  	/*
1447  	 * Size the pcf array based on the number of cpus in the box at
1448  	 * boot time.
1449  	 */
1450  	pcf_init();
1451  
1452  	/*
1453  	 * Initialize the page structures from the memory lists.
1454  	 */
1455  	kphysm_init();
1456  
1457  	availrmem_initial = availrmem = freemem;
1458  	PRM_DEBUG(availrmem);
1459  
1460  	/*
1461  	 * Some of the locks depend on page_hashsz being set!
1462  	 * kmem_init() depends on this; so, keep it here.
1463  	 */
1464  	page_lock_init();
1465  
1466  	/*
1467  	 * Initialize kernel memory allocator.
1468  	 */
1469  	kmem_init();
1470  
1471  	/*
1472  	 * Factor in colorequiv to check additional 'equivalent' bins
1473  	 */
1474  	if (&page_set_colorequiv_arr_cpu != NULL)
1475  		page_set_colorequiv_arr_cpu();
1476  	else
1477  		page_set_colorequiv_arr();
1478  
1479  	/*
1480  	 * Initialize bp_mapin().
1481  	 */
1482  	bp_init(shm_alignment, HAT_STRICTORDER);
1483  
1484  	/*
1485  	 * Reserve space for MPO mblock structs from the 32-bit heap.
1486  	 */
1487  
1488  	if (mpo_heap32_bufsz > (size_t)0) {
1489  		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1490  		    PAGESIZE, 0, 0, mpo_heap32_buf,
1491  		    mpo_heap32_buf + mpo_heap32_bufsz,
1492  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1493  	}
1494  	mem_config_init();
1495  }
1496  
1497  static void
1498  startup_modules(void)
1499  {
1500  	int nhblk1, nhblk8;
1501  	size_t  nhblksz;
1502  	pgcnt_t pages_per_hblk;
1503  	size_t hme8blk_sz, hme1blk_sz;
1504  
1505  	/*
1506  	 * The system file /etc/system was read already under startup_memlist.
1507  	 */
1508  	if (&set_platform_defaults)
1509  		set_platform_defaults();
1510  
1511  	/*
1512  	 * Calculate default settings of system parameters based upon
1513  	 * maxusers, yet allow to be overridden via the /etc/system file.
1514  	 */
1515  	param_calc(0);
1516  
1517  	mod_setup();
1518  
1519  	/*
1520  	 * If we are running firmware that isn't 64-bit ready
1521  	 * then complain and halt.
1522  	 */
1523  	do_prom_version_check();
1524  
1525  	/*
1526  	 * Initialize system parameters
1527  	 */
1528  	param_init();
1529  
1530  	/*
1531  	 * maxmem is the amount of physical memory we're playing with.
1532  	 */
1533  	maxmem = physmem;
1534  
1535  	/* Set segkp limits. */
1536  	ncbase = kdi_segdebugbase;
1537  	ncend = kdi_segdebugbase;
1538  
1539  	/*
1540  	 * Initialize the hat layer.
1541  	 */
1542  	hat_init();
1543  
1544  	/*
1545  	 * Initialize segment management stuff.
1546  	 */
1547  	seg_init();
1548  
1549  	/*
1550  	 * Create the va>tte handler, so the prom can understand
1551  	 * kernel translations.  The handler is installed later, just
1552  	 * as we are about to take over the trap table from the prom.
1553  	 */
1554  	create_va_to_tte();
1555  
1556  	/*
1557  	 * Load the forthdebugger (optional)
1558  	 */
1559  	forthdebug_init();
1560  
1561  	/*
1562  	 * Create OBP node for console input callbacks
1563  	 * if it is needed.
1564  	 */
1565  	startup_create_io_node();
1566  
1567  	if (modloadonly("fs", "specfs") == -1)
1568  		halt("Can't load specfs");
1569  
1570  	if (modloadonly("fs", "devfs") == -1)
1571  		halt("Can't load devfs");
1572  
1573  	if (modloadonly("fs", "procfs") == -1)
1574  		halt("Can't load procfs");
1575  
1576  	if (modloadonly("misc", "swapgeneric") == -1)
1577  		halt("Can't load swapgeneric");
1578  
1579  	(void) modloadonly("sys", "lbl_edition");
1580  
1581  	dispinit();
1582  
1583  	/*
1584  	 * Infer meanings to the members of the idprom buffer.
1585  	 */
1586  	parse_idprom();
1587  
1588  	/* Read cluster configuration data. */
1589  	clconf_init();
1590  
1591  	setup_ddi();
1592  
1593  	/*
1594  	 * Lets take this opportunity to load the root device.
1595  	 */
1596  	if (loadrootmodules() != 0)
1597  		debug_enter("Can't load the root filesystem");
1598  
1599  	/*
1600  	 * Load tod driver module for the tod part found on this system.
1601  	 * Recompute the cpu frequency/delays based on tod as tod part
1602  	 * tends to keep time more accurately.
1603  	 */
1604  	if (&load_tod_module)
1605  		load_tod_module();
1606  
1607  	/*
1608  	 * Allow platforms to load modules which might
1609  	 * be needed after bootops are gone.
1610  	 */
1611  	if (&load_platform_modules)
1612  		load_platform_modules();
1613  
1614  	setcpudelay();
1615  
1616  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1617  	    &boot_physavail, &boot_physavail_len,
1618  	    &boot_virtavail, &boot_virtavail_len);
1619  
1620  	/*
1621  	 * Calculation and allocation of hmeblks needed to remap
1622  	 * the memory allocated by PROM till now.
1623  	 * Overestimate the number of hblk1 elements by assuming
1624  	 * worst case of TTE64K mappings.
1625  	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1626  	 */
1627  	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1628  	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1629  	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1630  	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1631  
1632  	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1633  
1634  	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1635  	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1636  	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1637  
1638  	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1639  
1640  	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1641  	nhblk8 = 0;
1642  	while (bop_alloc_pages > 1) {
1643  		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1644  		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1645  		bop_alloc_pages *= hme8blk_sz;
1646  		bop_alloc_pages = btopr(bop_alloc_pages);
1647  	}
1648  	nhblk8 += 2;
1649  
1650  	/*
1651  	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1652  	 * boundary, the number of hblk8's needed to map the entries in the
1653  	 * boot_virtavail list needs to be adjusted to take this into
1654  	 * consideration.  Thus, we need to add additional hblk8's since it
1655  	 * is possible that an hblk8 will not have all 8 slots used due to
1656  	 * alignment constraints.  Since there were boot_virtavail_len entries
1657  	 * in that list, we need to add that many hblk8's to the number
1658  	 * already calculated to make sure we don't underestimate.
1659  	 */
1660  	nhblk8 += boot_virtavail_len;
1661  	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1662  
1663  	/* Allocate in pagesize chunks */
1664  	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1665  	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1666  	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1667  }
1668  
1669  static void
1670  startup_bop_gone(void)
1671  {
1672  
1673  	/*
1674  	 * Destroy the MD initialized at startup
1675  	 * The startup initializes the MD framework
1676  	 * using prom and BOP alloc free it now.
1677  	 */
1678  	mach_descrip_startup_fini();
1679  
1680  	/*
1681  	 * We're done with prom allocations.
1682  	 */
1683  	bop_fini();
1684  
1685  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1686  	    &boot_physavail, &boot_physavail_len,
1687  	    &boot_virtavail, &boot_virtavail_len);
1688  
1689  	/*
1690  	 * setup physically contiguous area twice as large as the ecache.
1691  	 * this is used while doing displacement flush of ecaches
1692  	 */
1693  	if (&ecache_flush_address) {
1694  		ecache_flushaddr = ecache_flush_address();
1695  		if (ecache_flushaddr == (uint64_t)-1) {
1696  			cmn_err(CE_PANIC,
1697  			    "startup: no memory to set ecache_flushaddr");
1698  		}
1699  	}
1700  
1701  	/*
1702  	 * Virtual available next.
1703  	 */
1704  	ASSERT(virt_avail != NULL);
1705  	memlist_free_list(virt_avail);
1706  	virt_avail = memlist;
1707  	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1708  
1709  }
1710  
1711  
1712  /*
1713   * startup_fixup_physavail - called from mach_sfmmu.c after the final
1714   * allocations have been performed.  We can't call it in startup_bop_gone
1715   * since later operations can cause obp to allocate more memory.
1716   */
1717  void
1718  startup_fixup_physavail(void)
1719  {
1720  	struct memlist *cur;
1721  	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1722  
1723  	PRM_DEBUG(kmem64_overmap_size);
1724  
1725  	/*
1726  	 * take the most current snapshot we can by calling mem-update
1727  	 */
1728  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1729  	    &boot_physavail, &boot_physavail_len,
1730  	    &boot_virtavail, &boot_virtavail_len);
1731  
1732  	/*
1733  	 * Copy phys_avail list, again.
1734  	 * Both the kernel/boot and the prom have been allocating
1735  	 * from the original list we copied earlier.
1736  	 */
1737  	cur = memlist;
1738  	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1739  
1740  	/*
1741  	 * Add any unused kmem64 memory from overmapped page
1742  	 * (Note: va_to_pa does not work for kmem64_end)
1743  	 */
1744  	if (kmem64_overmap_size) {
1745  		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1746  		    kmem64_overmap_size, &memlist, &cur);
1747  	}
1748  
1749  	/*
1750  	 * Add any extra memory after e_data we added to the phys_avail list
1751  	 * back to the old list.
1752  	 */
1753  	if (ndata_remain_sz >= MMU_PAGESIZE)
1754  		memlist_add(va_to_pa(nalloc_base),
1755  		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1756  
1757  	/*
1758  	 * There isn't any bounds checking on the memlist area
1759  	 * so ensure it hasn't overgrown.
1760  	 */
1761  	if ((caddr_t)memlist > (caddr_t)memlist_end)
1762  		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1763  
1764  	/*
1765  	 * The kernel removes the pages that were allocated for it from
1766  	 * the freelist, but we now have to find any -extra- pages that
1767  	 * the prom has allocated for it's own book-keeping, and remove
1768  	 * them from the freelist too. sigh.
1769  	 */
1770  	sync_memlists(phys_avail, cur);
1771  
1772  	ASSERT(phys_avail != NULL);
1773  
1774  	old_phys_avail = phys_avail;
1775  	phys_avail = cur;
1776  }
1777  
1778  void
1779  update_kcage_ranges(uint64_t addr, uint64_t len)
1780  {
1781  	pfn_t base = btop(addr);
1782  	pgcnt_t num = btop(len);
1783  	int rv;
1784  
1785  	rv = kcage_range_add(base, num, kcage_startup_dir);
1786  
1787  	if (rv == ENOMEM) {
1788  		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1789  		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1790  	} else if (rv != 0) {
1791  		/* catch this in debug kernels */
1792  		ASSERT(0);
1793  
1794  		cmn_err(CE_WARN, "unexpected kcage_range_add"
1795  		    " return value %d", rv);
1796  	}
1797  }
1798  
1799  static void
1800  startup_vm(void)
1801  {
1802  	size_t	i;
1803  	struct segmap_crargs a;
1804  	struct segkpm_crargs b;
1805  
1806  	uint64_t avmem;
1807  	caddr_t va;
1808  	pgcnt_t	max_phys_segkp;
1809  	int	mnode;
1810  
1811  	extern int use_brk_lpg, use_stk_lpg;
1812  
1813  	/*
1814  	 * get prom's mappings, create hments for them and switch
1815  	 * to the kernel context.
1816  	 */
1817  	hat_kern_setup();
1818  
1819  	/*
1820  	 * Take over trap table
1821  	 */
1822  	setup_trap_table();
1823  
1824  	/*
1825  	 * Install the va>tte handler, so that the prom can handle
1826  	 * misses and understand the kernel table layout in case
1827  	 * we need call into the prom.
1828  	 */
1829  	install_va_to_tte();
1830  
1831  	/*
1832  	 * Set a flag to indicate that the tba has been taken over.
1833  	 */
1834  	tba_taken_over = 1;
1835  
1836  	/* initialize MMU primary context register */
1837  	mmu_init_kcontext();
1838  
1839  	/*
1840  	 * The boot cpu can now take interrupts, x-calls, x-traps
1841  	 */
1842  	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1843  	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1844  
1845  	/*
1846  	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1847  	 */
1848  	tbr_wr_addr_inited = 1;
1849  
1850  	/*
1851  	 * Initialize VM system, and map kernel address space.
1852  	 */
1853  	kvm_init();
1854  
1855  	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1856  	if (kernel_cage_enable) {
1857  		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1858  	}
1859  	memlist_free_list(old_phys_avail);
1860  
1861  	/*
1862  	 * If the following is true, someone has patched
1863  	 * phsymem to be less than the number of pages that
1864  	 * the system actually has.  Remove pages until system
1865  	 * memory is limited to the requested amount.  Since we
1866  	 * have allocated page structures for all pages, we
1867  	 * correct the amount of memory we want to remove
1868  	 * by the size of the memory used to hold page structures
1869  	 * for the non-used pages.
1870  	 */
1871  	if (physmem + ramdisk_npages < npages) {
1872  		pgcnt_t diff, off;
1873  		struct page *pp;
1874  		struct seg kseg;
1875  
1876  		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1877  
1878  		off = 0;
1879  		diff = npages - (physmem + ramdisk_npages);
1880  		diff -= mmu_btopr(diff * sizeof (struct page));
1881  		kseg.s_as = &kas;
1882  		while (diff--) {
1883  			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1884  			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1885  			    &kseg, (caddr_t)off);
1886  			if (pp == NULL)
1887  				cmn_err(CE_PANIC, "limited physmem too much!");
1888  			page_io_unlock(pp);
1889  			page_downgrade(pp);
1890  			availrmem--;
1891  			off += MMU_PAGESIZE;
1892  		}
1893  	}
1894  
1895  	/*
1896  	 * When printing memory, show the total as physmem less
1897  	 * that stolen by a debugger.
1898  	 */
1899  	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1900  	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1901  	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1902  
1903  	avmem = (uint64_t)freemem << PAGESHIFT;
1904  	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1905  
1906  	/*
1907  	 * For small memory systems disable automatic large pages.
1908  	 */
1909  	if (physmem < privm_lpg_min_physmem) {
1910  		use_brk_lpg = 0;
1911  		use_stk_lpg = 0;
1912  	}
1913  
1914  	/*
1915  	 * Perform platform specific freelist processing
1916  	 */
1917  	if (&plat_freelist_process) {
1918  		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1919  			if (mem_node_config[mnode].exists)
1920  				plat_freelist_process(mnode);
1921  	}
1922  
1923  	/*
1924  	 * Initialize the segkp segment type.  We position it
1925  	 * after the configured tables and buffers (whose end
1926  	 * is given by econtig) and before V_WKBASE_ADDR.
1927  	 * Also in this area is segkmap (size SEGMAPSIZE).
1928  	 */
1929  
1930  	/* XXX - cache alignment? */
1931  	va = (caddr_t)SEGKPBASE;
1932  	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1933  
1934  	max_phys_segkp = (physmem * 2);
1935  
1936  	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1937  		segkpsize = btop(SEGKPDEFSIZE);
1938  		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1939  		    "segkpsize has been reset to %ld pages", segkpsize);
1940  	}
1941  
1942  	i = ptob(MIN(segkpsize, max_phys_segkp));
1943  
1944  	rw_enter(&kas.a_lock, RW_WRITER);
1945  	if (seg_attach(&kas, va, i, segkp) < 0)
1946  		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1947  	if (segkp_create(segkp) != 0)
1948  		cmn_err(CE_PANIC, "startup: segkp_create failed");
1949  	rw_exit(&kas.a_lock);
1950  
1951  	/*
1952  	 * kpm segment
1953  	 */
1954  	segmap_kpm = kpm_enable &&
1955  	    segmap_kpm && PAGESIZE == MAXBSIZE;
1956  
1957  	if (kpm_enable) {
1958  		rw_enter(&kas.a_lock, RW_WRITER);
1959  
1960  		/*
1961  		 * The segkpm virtual range range is larger than the
1962  		 * actual physical memory size and also covers gaps in
1963  		 * the physical address range for the following reasons:
1964  		 * . keep conversion between segkpm and physical addresses
1965  		 *   simple, cheap and unambiguous.
1966  		 * . avoid extension/shrink of the the segkpm in case of DR.
1967  		 * . avoid complexity for handling of virtual addressed
1968  		 *   caches, segkpm and the regular mapping scheme must be
1969  		 *   kept in sync wrt. the virtual color of mapped pages.
1970  		 * Any accesses to virtual segkpm ranges not backed by
1971  		 * physical memory will fall through the memseg pfn hash
1972  		 * and will be handled in segkpm_fault.
1973  		 * Additional kpm_size spaces needed for vac alias prevention.
1974  		 */
1975  		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1976  		    segkpm) < 0)
1977  			cmn_err(CE_PANIC, "cannot attach segkpm");
1978  
1979  		b.prot = PROT_READ | PROT_WRITE;
1980  		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1981  
1982  		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1983  			panic("segkpm_create segkpm");
1984  
1985  		rw_exit(&kas.a_lock);
1986  
1987  		mach_kpm_init();
1988  	}
1989  
1990  	va = kpm_vbase + (kpm_size * vac_colors);
1991  
1992  	if (!segzio_fromheap) {
1993  		size_t size;
1994  		size_t physmem_b = mmu_ptob(physmem);
1995  
1996  		/* size is in bytes, segziosize is in pages */
1997  		if (segziosize == 0) {
1998  			size = physmem_b;
1999  		} else {
2000  			size = mmu_ptob(segziosize);
2001  		}
2002  
2003  		if (size < SEGZIOMINSIZE) {
2004  			size = SEGZIOMINSIZE;
2005  		} else if (size > SEGZIOMAXSIZE) {
2006  			size = SEGZIOMAXSIZE;
2007  			/*
2008  			 * On 64-bit x86, we only have 2TB of KVA.  This exists
2009  			 * for parity with x86.
2010  			 *
2011  			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2012  			 * doesn't consume all of KVA.  However, if we have a
2013  			 * system that has more thant 512gb of physical memory,
2014  			 * we can actually consume about half of the difference
2015  			 * between 512gb and the rest of the available physical
2016  			 * memory.
2017  			 */
2018  			if (physmem_b > SEGZIOMAXSIZE) {
2019  				size += (physmem_b - SEGZIOMAXSIZE) / 2;
2020  		}
2021  		}
2022  		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2023  		/* put the base of the ZIO segment after the kpm segment */
2024  		segzio_base = va;
2025  		va += mmu_ptob(segziosize);
2026  		PRM_DEBUG(segziosize);
2027  		PRM_DEBUG(segzio_base);
2028  
2029  		/*
2030  		 * On some platforms, kvm_init is called after the kpm
2031  		 * sizes have been determined.  On SPARC, kvm_init is called
2032  		 * before, so we have to attach the kzioseg after kvm is
2033  		 * initialized, otherwise we'll try to allocate from the boot
2034  		 * area since the kernel heap hasn't yet been configured.
2035  		 */
2036  		rw_enter(&kas.a_lock, RW_WRITER);
2037  
2038  		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2039  		    &kzioseg);
2040  		(void) segkmem_zio_create(&kzioseg);
2041  
2042  		/* create zio area covering new segment */
2043  		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2044  
2045  		rw_exit(&kas.a_lock);
2046  	}
2047  
2048  	if (ppvm_enable) {
2049  		caddr_t ppvm_max;
2050  
2051  		/*
2052  		 * ppvm refers to the static VA space used to map
2053  		 * the page_t's for dynamically added memory.
2054  		 *
2055  		 * ppvm_base should not cross a potential VA hole.
2056  		 *
2057  		 * ppvm_size should be large enough to map the
2058  		 * page_t's needed to manage all of KPM range.
2059  		 */
2060  		ppvm_size =
2061  		    roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2062  		    MMU_PAGESIZE);
2063  		ppvm_max = (caddr_t)(0ull - ppvm_size);
2064  		ppvm_base = (page_t *)va;
2065  
2066  		if ((caddr_t)ppvm_base <= hole_end) {
2067  			cmn_err(CE_WARN,
2068  			    "Memory DR disabled: invalid DR map base: 0x%p\n",
2069  			    (void *)ppvm_base);
2070  			ppvm_enable = 0;
2071  		} else if ((caddr_t)ppvm_base > ppvm_max) {
2072  			uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2073  
2074  			cmn_err(CE_WARN,
2075  			    "Memory DR disabled: insufficient DR map size:"
2076  			    " 0x%lx (needed 0x%lx)\n",
2077  			    ppvm_size - diff, ppvm_size);
2078  			ppvm_enable = 0;
2079  		}
2080  		PRM_DEBUG(ppvm_size);
2081  		PRM_DEBUG(ppvm_base);
2082  	}
2083  
2084  	/*
2085  	 * Now create generic mapping segment.  This mapping
2086  	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2087  	 * virtual address is greater than the amount of free
2088  	 * memory that is available, then we trim back the
2089  	 * segment size to that amount
2090  	 */
2091  	va = (caddr_t)SEGMAPBASE;
2092  
2093  	/*
2094  	 * 1201049: segkmap base address must be MAXBSIZE aligned
2095  	 */
2096  	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2097  
2098  	/*
2099  	 * Set size of segmap to percentage of freemem at boot,
2100  	 * but stay within the allowable range
2101  	 * Note we take percentage  before converting from pages
2102  	 * to bytes to avoid an overflow on 32-bit kernels.
2103  	 */
2104  	i = mmu_ptob((freemem * segmap_percent) / 100);
2105  
2106  	if (i < MINMAPSIZE)
2107  		i = MINMAPSIZE;
2108  
2109  	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2110  		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2111  
2112  	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
2113  
2114  	rw_enter(&kas.a_lock, RW_WRITER);
2115  	if (seg_attach(&kas, va, i, segkmap) < 0)
2116  		cmn_err(CE_PANIC, "cannot attach segkmap");
2117  
2118  	a.prot = PROT_READ | PROT_WRITE;
2119  	a.shmsize = shm_alignment;
2120  	a.nfreelist = 0;	/* use segmap driver defaults */
2121  
2122  	if (segmap_create(segkmap, (caddr_t)&a) != 0)
2123  		panic("segmap_create segkmap");
2124  	rw_exit(&kas.a_lock);
2125  
2126  	segdev_init();
2127  }
2128  
2129  static void
2130  startup_end(void)
2131  {
2132  	if ((caddr_t)memlist > (caddr_t)memlist_end)
2133  		panic("memlist overflow 2");
2134  	memlist_free_block((caddr_t)memlist,
2135  	    ((caddr_t)memlist_end - (caddr_t)memlist));
2136  	memlist = NULL;
2137  
2138  	/* enable page_relocation since OBP is now done */
2139  	page_relocate_ready = 1;
2140  
2141  	/*
2142  	 * Perform tasks that get done after most of the VM
2143  	 * initialization has been done but before the clock
2144  	 * and other devices get started.
2145  	 */
2146  	kern_setup1();
2147  
2148  	/*
2149  	 * Perform CPC initialization for this CPU.
2150  	 */
2151  	kcpc_hw_init();
2152  
2153  	/*
2154  	 * Intialize the VM arenas for allocating physically
2155  	 * contiguus memory chunk for interrupt queues snd
2156  	 * allocate/register boot cpu's queues, if any and
2157  	 * allocate dump buffer for sun4v systems to store
2158  	 * extra crash information during crash dump
2159  	 */
2160  	contig_mem_init();
2161  	mach_descrip_init();
2162  
2163  	if (cpu_intrq_setup(CPU)) {
2164  		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2165  	}
2166  	cpu_intrq_register(CPU);
2167  	mach_htraptrace_setup(CPU->cpu_id);
2168  	mach_htraptrace_configure(CPU->cpu_id);
2169  	mach_dump_buffer_init();
2170  
2171  	/*
2172  	 * Initialize interrupt related stuff
2173  	 */
2174  	cpu_intr_alloc(CPU, NINTR_THREADS);
2175  
2176  	(void) splzs();			/* allow hi clock ints but not zs */
2177  
2178  	/*
2179  	 * Initialize errors.
2180  	 */
2181  	error_init();
2182  
2183  	/*
2184  	 * Note that we may have already used kernel bcopy before this
2185  	 * point - but if you really care about this, adb the use_hw_*
2186  	 * variables to 0 before rebooting.
2187  	 */
2188  	mach_hw_copy_limit();
2189  
2190  	/*
2191  	 * Install the "real" preemption guards before DDI services
2192  	 * are available.
2193  	 */
2194  	(void) prom_set_preprom(kern_preprom);
2195  	(void) prom_set_postprom(kern_postprom);
2196  	CPU->cpu_m.mutex_ready = 1;
2197  
2198  	/*
2199  	 * Initialize segnf (kernel support for non-faulting loads).
2200  	 */
2201  	segnf_init();
2202  
2203  	/*
2204  	 * Configure the root devinfo node.
2205  	 */
2206  	configure();		/* set up devices */
2207  	mach_cpu_halt_idle();
2208  }
2209  
2210  
2211  void
2212  post_startup(void)
2213  {
2214  #ifdef	PTL1_PANIC_DEBUG
2215  	extern void init_ptl1_thread(void);
2216  #endif	/* PTL1_PANIC_DEBUG */
2217  	extern void abort_sequence_init(void);
2218  
2219  	/*
2220  	 * Set the system wide, processor-specific flags to be passed
2221  	 * to userland via the aux vector for performance hints and
2222  	 * instruction set extensions.
2223  	 */
2224  	bind_hwcap();
2225  
2226  	/*
2227  	 * Startup memory scrubber (if any)
2228  	 */
2229  	mach_memscrub();
2230  
2231  	/*
2232  	 * Allocate soft interrupt to handle abort sequence.
2233  	 */
2234  	abort_sequence_init();
2235  
2236  	/*
2237  	 * Configure the rest of the system.
2238  	 * Perform forceloading tasks for /etc/system.
2239  	 */
2240  	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2241  	/*
2242  	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2243  	 * ON4.0: This must be fixed or restated in /etc/systems.
2244  	 */
2245  	(void) modload("fs", "procfs");
2246  
2247  	/* load machine class specific drivers */
2248  	load_mach_drivers();
2249  
2250  	/* load platform specific drivers */
2251  	if (&load_platform_drivers)
2252  		load_platform_drivers();
2253  
2254  	/* load vis simulation module, if we are running w/fpu off */
2255  	if (!fpu_exists) {
2256  		if (modload("misc", "vis") == -1)
2257  			halt("Can't load vis");
2258  	}
2259  
2260  	mach_fpras();
2261  
2262  	maxmem = freemem;
2263  
2264  	pg_init();
2265  
2266  #ifdef	PTL1_PANIC_DEBUG
2267  	init_ptl1_thread();
2268  #endif	/* PTL1_PANIC_DEBUG */
2269  }
2270  
2271  #ifdef	PTL1_PANIC_DEBUG
2272  int		ptl1_panic_test = 0;
2273  int		ptl1_panic_xc_one_test = 0;
2274  int		ptl1_panic_xc_all_test = 0;
2275  int		ptl1_panic_xt_one_test = 0;
2276  int		ptl1_panic_xt_all_test = 0;
2277  kthread_id_t	ptl1_thread_p = NULL;
2278  kcondvar_t	ptl1_cv;
2279  kmutex_t	ptl1_mutex;
2280  int		ptl1_recurse_count_threshold = 0x40;
2281  int		ptl1_recurse_trap_threshold = 0x3d;
2282  extern void	ptl1_recurse(int, int);
2283  extern void	ptl1_panic_xt(int, int);
2284  
2285  /*
2286   * Called once per second by timeout() to wake up
2287   * the ptl1_panic thread to see if it should cause
2288   * a trap to the ptl1_panic() code.
2289   */
2290  /* ARGSUSED */
2291  static void
2292  ptl1_wakeup(void *arg)
2293  {
2294  	mutex_enter(&ptl1_mutex);
2295  	cv_signal(&ptl1_cv);
2296  	mutex_exit(&ptl1_mutex);
2297  }
2298  
2299  /*
2300   * ptl1_panic cross call function:
2301   *     Needed because xc_one() and xc_some() can pass
2302   *	64 bit args but ptl1_recurse() expects ints.
2303   */
2304  static void
2305  ptl1_panic_xc(void)
2306  {
2307  	ptl1_recurse(ptl1_recurse_count_threshold,
2308  	    ptl1_recurse_trap_threshold);
2309  }
2310  
2311  /*
2312   * The ptl1 thread waits for a global flag to be set
2313   * and uses the recurse thresholds to set the stack depth
2314   * to cause a ptl1_panic() directly via a call to ptl1_recurse
2315   * or indirectly via the cross call and cross trap functions.
2316   *
2317   * This is useful testing stack overflows and normal
2318   * ptl1_panic() states with a know stack frame.
2319   *
2320   * ptl1_recurse() is an asm function in ptl1_panic.s that
2321   * sets the {In, Local, Out, and Global} registers to a
2322   * know state on the stack and just prior to causing a
2323   * test ptl1_panic trap.
2324   */
2325  static void
2326  ptl1_thread(void)
2327  {
2328  	mutex_enter(&ptl1_mutex);
2329  	while (ptl1_thread_p) {
2330  		cpuset_t	other_cpus;
2331  		int		cpu_id;
2332  		int		my_cpu_id;
2333  		int		target_cpu_id;
2334  		int		target_found;
2335  
2336  		if (ptl1_panic_test) {
2337  			ptl1_recurse(ptl1_recurse_count_threshold,
2338  			    ptl1_recurse_trap_threshold);
2339  		}
2340  
2341  		/*
2342  		 * Find potential targets for x-call and x-trap,
2343  		 * if any exist while preempt is disabled we
2344  		 * start a ptl1_panic if requested via a
2345  		 * globals.
2346  		 */
2347  		kpreempt_disable();
2348  		my_cpu_id = CPU->cpu_id;
2349  		other_cpus = cpu_ready_set;
2350  		CPUSET_DEL(other_cpus, CPU->cpu_id);
2351  		target_found = 0;
2352  		if (!CPUSET_ISNULL(other_cpus)) {
2353  			/*
2354  			 * Pick the first one
2355  			 */
2356  			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2357  				if (cpu_id == my_cpu_id)
2358  					continue;
2359  
2360  				if (CPU_XCALL_READY(cpu_id)) {
2361  					target_cpu_id = cpu_id;
2362  					target_found = 1;
2363  					break;
2364  				}
2365  			}
2366  			ASSERT(target_found);
2367  
2368  			if (ptl1_panic_xc_one_test) {
2369  				xc_one(target_cpu_id,
2370  				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2371  			}
2372  			if (ptl1_panic_xc_all_test) {
2373  				xc_some(other_cpus,
2374  				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2375  			}
2376  			if (ptl1_panic_xt_one_test) {
2377  				xt_one(target_cpu_id,
2378  				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2379  			}
2380  			if (ptl1_panic_xt_all_test) {
2381  				xt_some(other_cpus,
2382  				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2383  			}
2384  		}
2385  		kpreempt_enable();
2386  		(void) timeout(ptl1_wakeup, NULL, hz);
2387  		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2388  	}
2389  	mutex_exit(&ptl1_mutex);
2390  }
2391  
2392  /*
2393   * Called during early startup to create the ptl1_thread
2394   */
2395  void
2396  init_ptl1_thread(void)
2397  {
2398  	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2399  	    &p0, TS_RUN, 0);
2400  }
2401  #endif	/* PTL1_PANIC_DEBUG */
2402  
2403  
2404  static void
2405  memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2406  {
2407  	struct memlist *new;
2408  
2409  	new = *memlistp;
2410  	new->ml_address = start;
2411  	new->ml_size = len;
2412  	*memlistp = new + 1;
2413  }
2414  
2415  /*
2416   * Add to a memory list.
2417   * start = start of new memory segment
2418   * len = length of new memory segment in bytes
2419   * memlistp = pointer to array of available memory segment structures
2420   * curmemlistp = memory list to which to add segment.
2421   */
2422  static void
2423  memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2424      struct memlist **curmemlistp)
2425  {
2426  	struct memlist *new = *memlistp;
2427  
2428  	memlist_new(start, len, memlistp);
2429  	memlist_insert(new, curmemlistp);
2430  }
2431  
2432  static int
2433  ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2434  {
2435  	int nseg;
2436  	size_t memseg_sz;
2437  	struct memseg *msp;
2438  
2439  	/*
2440  	 * The memseg list is for the chunks of physical memory that
2441  	 * will be managed by the vm system.  The number calculated is
2442  	 * a guess as boot may fragment it more when memory allocations
2443  	 * are made before kphysm_init().
2444  	 */
2445  	memseg_sz = (avail + 10) * sizeof (struct memseg);
2446  	memseg_sz = roundup(memseg_sz, PAGESIZE);
2447  	nseg = memseg_sz / sizeof (struct memseg);
2448  	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2449  	if (msp == NULL)
2450  		return (1);
2451  	PRM_DEBUG(memseg_free);
2452  
2453  	while (nseg--) {
2454  		msp->next = memseg_free;
2455  		memseg_free = msp;
2456  		msp++;
2457  	}
2458  	return (0);
2459  }
2460  
2461  /*
2462   * In the case of architectures that support dynamic addition of
2463   * memory at run-time there are two cases where memsegs need to
2464   * be initialized and added to the memseg list.
2465   * 1) memsegs that are constructed at startup.
2466   * 2) memsegs that are constructed at run-time on
2467   *    hot-plug capable architectures.
2468   * This code was originally part of the function kphysm_init().
2469   */
2470  
2471  static void
2472  memseg_list_add(struct memseg *memsegp)
2473  {
2474  	struct memseg **prev_memsegp;
2475  	pgcnt_t num;
2476  
2477  	/* insert in memseg list, decreasing number of pages order */
2478  
2479  	num = MSEG_NPAGES(memsegp);
2480  
2481  	for (prev_memsegp = &memsegs; *prev_memsegp;
2482  	    prev_memsegp = &((*prev_memsegp)->next)) {
2483  		if (num > MSEG_NPAGES(*prev_memsegp))
2484  			break;
2485  	}
2486  
2487  	memsegp->next = *prev_memsegp;
2488  	*prev_memsegp = memsegp;
2489  
2490  	if (kpm_enable) {
2491  		memsegp->nextpa = (memsegp->next) ?
2492  		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2493  
2494  		if (prev_memsegp != &memsegs) {
2495  			struct memseg *msp;
2496  			msp = (struct memseg *)((caddr_t)prev_memsegp -
2497  			    offsetof(struct memseg, next));
2498  			msp->nextpa = va_to_pa(memsegp);
2499  		} else {
2500  			memsegspa = va_to_pa(memsegs);
2501  		}
2502  	}
2503  }
2504  
2505  /*
2506   * PSM add_physmem_cb(). US-II and newer processors have some
2507   * flavor of the prefetch capability implemented. We exploit
2508   * this capability for optimum performance.
2509   */
2510  #define	PREFETCH_BYTES	64
2511  
2512  void
2513  add_physmem_cb(page_t *pp, pfn_t pnum)
2514  {
2515  	extern void	 prefetch_page_w(void *);
2516  
2517  	pp->p_pagenum = pnum;
2518  
2519  	/*
2520  	 * Prefetch one more page_t into E$. To prevent future
2521  	 * mishaps with the sizeof(page_t) changing on us, we
2522  	 * catch this on debug kernels if we can't bring in the
2523  	 * entire hpage with 2 PREFETCH_BYTES reads. See
2524  	 * also, sun4u/cpu/cpu_module.c
2525  	 */
2526  	/*LINTED*/
2527  	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2528  	prefetch_page_w((char *)pp);
2529  }
2530  
2531  /*
2532   * Find memseg with given pfn
2533   */
2534  static struct memseg *
2535  memseg_find(pfn_t base, pfn_t *next)
2536  {
2537  	struct memseg *seg;
2538  
2539  	if (next != NULL)
2540  		*next = LONG_MAX;
2541  	for (seg = memsegs; seg != NULL; seg = seg->next) {
2542  		if (base >= seg->pages_base && base < seg->pages_end)
2543  			return (seg);
2544  		if (next != NULL && seg->pages_base > base &&
2545  		    seg->pages_base < *next)
2546  			*next = seg->pages_base;
2547  	}
2548  	return (NULL);
2549  }
2550  
2551  /*
2552   * Put page allocated by OBP on prom_ppages
2553   */
2554  static void
2555  kphysm_erase(uint64_t addr, uint64_t len)
2556  {
2557  	struct page *pp;
2558  	struct memseg *seg;
2559  	pfn_t base = btop(addr), next;
2560  	pgcnt_t num = btop(len);
2561  
2562  	while (num != 0) {
2563  		pgcnt_t off, left;
2564  
2565  		seg = memseg_find(base, &next);
2566  		if (seg == NULL) {
2567  			if (next == LONG_MAX)
2568  				break;
2569  			left = MIN(next - base, num);
2570  			base += left, num -= left;
2571  			continue;
2572  		}
2573  		off = base - seg->pages_base;
2574  		pp = seg->pages + off;
2575  		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2576  		while (num != left) {
2577  			/*
2578  			 * init it, lock it, and hashin on prom_pages vp.
2579  			 *
2580  			 * Mark it as NONRELOC to let DR know the page
2581  			 * is locked long term, otherwise DR hangs when
2582  			 * trying to remove those pages.
2583  			 *
2584  			 * XXX	vnode offsets on the prom_ppages vnode
2585  			 *	are page numbers (gack) for >32 bit
2586  			 *	physical memory machines.
2587  			 */
2588  			PP_SETNORELOC(pp);
2589  			add_physmem_cb(pp, base);
2590  			if (page_trylock(pp, SE_EXCL) == 0)
2591  				cmn_err(CE_PANIC, "prom page locked");
2592  			(void) page_hashin(pp, &promvp,
2593  			    (offset_t)base, NULL);
2594  			(void) page_pp_lock(pp, 0, 1);
2595  			pp++, base++, num--;
2596  		}
2597  	}
2598  }
2599  
2600  static page_t *ppnext;
2601  static pgcnt_t ppleft;
2602  
2603  static void *kpm_ppnext;
2604  static pgcnt_t kpm_ppleft;
2605  
2606  /*
2607   * Create a memseg
2608   */
2609  static void
2610  kphysm_memseg(uint64_t addr, uint64_t len)
2611  {
2612  	pfn_t base = btop(addr);
2613  	pgcnt_t num = btop(len);
2614  	struct memseg *seg;
2615  
2616  	seg = memseg_free;
2617  	memseg_free = seg->next;
2618  	ASSERT(seg != NULL);
2619  
2620  	seg->pages = ppnext;
2621  	seg->epages = ppnext + num;
2622  	seg->pages_base = base;
2623  	seg->pages_end = base + num;
2624  	ppnext += num;
2625  	ppleft -= num;
2626  
2627  	if (kpm_enable) {
2628  		pgcnt_t kpnum = ptokpmpr(num);
2629  
2630  		if (kpnum > kpm_ppleft)
2631  			panic("kphysm_memseg: kpm_pp overflow");
2632  		seg->pagespa = va_to_pa(seg->pages);
2633  		seg->epagespa = va_to_pa(seg->epages);
2634  		seg->kpm_pbase = kpmptop(ptokpmp(base));
2635  		seg->kpm_nkpmpgs = kpnum;
2636  		/*
2637  		 * In the kpm_smallpage case, the kpm array
2638  		 * is 1-1 wrt the page array
2639  		 */
2640  		if (kpm_smallpages) {
2641  			kpm_spage_t *kpm_pp = kpm_ppnext;
2642  
2643  			kpm_ppnext = kpm_pp + kpnum;
2644  			seg->kpm_spages = kpm_pp;
2645  			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2646  		} else {
2647  			kpm_page_t *kpm_pp = kpm_ppnext;
2648  
2649  			kpm_ppnext = kpm_pp + kpnum;
2650  			seg->kpm_pages = kpm_pp;
2651  			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2652  			/* ASSERT no kpm overlaps */
2653  			ASSERT(
2654  			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2655  			ASSERT(memseg_find(
2656  			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2657  		}
2658  		kpm_ppleft -= kpnum;
2659  	}
2660  
2661  	memseg_list_add(seg);
2662  }
2663  
2664  /*
2665   * Add range to free list
2666   */
2667  void
2668  kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2669  {
2670  	struct page *pp;
2671  	struct memseg *seg;
2672  	pfn_t base = btop(addr);
2673  	pgcnt_t num = btop(len);
2674  
2675  	seg = memseg_find(base, NULL);
2676  	ASSERT(seg != NULL);
2677  	pp = seg->pages + (base - seg->pages_base);
2678  
2679  	if (reclaim) {
2680  		struct page *rpp = pp;
2681  		struct page *lpp = pp + num;
2682  
2683  		/*
2684  		 * page should be locked on prom_ppages
2685  		 * unhash and unlock it
2686  		 */
2687  		while (rpp < lpp) {
2688  			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2689  			ASSERT(PP_ISNORELOC(rpp));
2690  			PP_CLRNORELOC(rpp);
2691  			page_pp_unlock(rpp, 0, 1);
2692  			page_hashout(rpp, NULL);
2693  			page_unlock(rpp);
2694  			rpp++;
2695  		}
2696  	}
2697  
2698  	/*
2699  	 * add_physmem() initializes the PSM part of the page
2700  	 * struct by calling the PSM back with add_physmem_cb().
2701  	 * In addition it coalesces pages into larger pages as
2702  	 * it initializes them.
2703  	 */
2704  	add_physmem(pp, num, base);
2705  }
2706  
2707  /*
2708   * kphysm_init() tackles the problem of initializing physical memory.
2709   */
2710  static void
2711  kphysm_init(void)
2712  {
2713  	struct memlist *pmem;
2714  
2715  	ASSERT(page_hash != NULL && page_hashsz != 0);
2716  
2717  	ppnext = pp_base;
2718  	ppleft = npages;
2719  	kpm_ppnext = kpm_pp_base;
2720  	kpm_ppleft = kpm_npages;
2721  
2722  	/*
2723  	 * installed pages not on nopp_memlist go in memseg list
2724  	 */
2725  	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2726  
2727  	/*
2728  	 * Free the avail list
2729  	 */
2730  	for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2731  		kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2732  
2733  	/*
2734  	 * Erase pages that aren't available
2735  	 */
2736  	diff_memlists(phys_install, phys_avail, kphysm_erase);
2737  
2738  	build_pfn_hash();
2739  }
2740  
2741  /*
2742   * Kernel VM initialization.
2743   * Assumptions about kernel address space ordering:
2744   *	(1) gap (user space)
2745   *	(2) kernel text
2746   *	(3) kernel data/bss
2747   *	(4) gap
2748   *	(5) kernel data structures
2749   *	(6) gap
2750   *	(7) debugger (optional)
2751   *	(8) monitor
2752   *	(9) gap (possibly null)
2753   *	(10) dvma
2754   *	(11) devices
2755   */
2756  static void
2757  kvm_init(void)
2758  {
2759  	/*
2760  	 * Put the kernel segments in kernel address space.
2761  	 */
2762  	rw_enter(&kas.a_lock, RW_WRITER);
2763  	as_avlinit(&kas);
2764  
2765  	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2766  	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2767  	(void) segkmem_create(&ktextseg);
2768  
2769  	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2770  	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2771  	(void) segkmem_create(&ktexthole);
2772  
2773  	(void) seg_attach(&kas, (caddr_t)valloc_base,
2774  	    (size_t)(econtig32 - valloc_base), &kvalloc);
2775  	(void) segkmem_create(&kvalloc);
2776  
2777  	if (kmem64_base) {
2778  		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2779  		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2780  		(void) segkmem_create(&kmem64);
2781  	}
2782  
2783  	/*
2784  	 * We're about to map out /boot.  This is the beginning of the
2785  	 * system resource management transition. We can no longer
2786  	 * call into /boot for I/O or memory allocations.
2787  	 */
2788  	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2789  	(void) segkmem_create(&kvseg);
2790  	hblk_alloc_dynamic = 1;
2791  
2792  	/*
2793  	 * we need to preallocate pages for DR operations before enabling large
2794  	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2795  	 */
2796  	memseg_remap_init();
2797  
2798  	/* at this point we are ready to use large page heap */
2799  	segkmem_heap_lp_init();
2800  
2801  	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2802  	    &kvseg32);
2803  	(void) segkmem_create(&kvseg32);
2804  
2805  	/*
2806  	 * Create a segment for the debugger.
2807  	 */
2808  	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2809  	(void) segkmem_create(&kdebugseg);
2810  
2811  	rw_exit(&kas.a_lock);
2812  }
2813  
2814  char obp_tte_str[] =
2815  	"h# %x constant MMU_PAGESHIFT "
2816  	"h# %x constant TTE8K "
2817  	"h# %x constant SFHME_SIZE "
2818  	"h# %x constant SFHME_TTE "
2819  	"h# %x constant HMEBLK_TAG "
2820  	"h# %x constant HMEBLK_NEXT "
2821  	"h# %x constant HMEBLK_MISC "
2822  	"h# %x constant HMEBLK_HME1 "
2823  	"h# %x constant NHMENTS "
2824  	"h# %x constant HBLK_SZMASK "
2825  	"h# %x constant HBLK_RANGE_SHIFT "
2826  	"h# %x constant HMEBP_HBLK "
2827  	"h# %x constant HMEBLK_ENDPA "
2828  	"h# %x constant HMEBUCKET_SIZE "
2829  	"h# %x constant HTAG_SFMMUPSZ "
2830  	"h# %x constant HTAG_BSPAGE_SHIFT "
2831  	"h# %x constant HTAG_REHASH_SHIFT "
2832  	"h# %x constant SFMMU_INVALID_SHMERID "
2833  	"h# %x constant mmu_hashcnt "
2834  	"h# %p constant uhme_hash "
2835  	"h# %p constant khme_hash "
2836  	"h# %x constant UHMEHASH_SZ "
2837  	"h# %x constant KHMEHASH_SZ "
2838  	"h# %p constant KCONTEXT "
2839  	"h# %p constant KHATID "
2840  	"h# %x constant ASI_MEM "
2841  
2842  	": PHYS-X@ ( phys -- data ) "
2843  	"   ASI_MEM spacex@ "
2844  	"; "
2845  
2846  	": PHYS-W@ ( phys -- data ) "
2847  	"   ASI_MEM spacew@ "
2848  	"; "
2849  
2850  	": PHYS-L@ ( phys -- data ) "
2851  	"   ASI_MEM spaceL@ "
2852  	"; "
2853  
2854  	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2855  	"   3 * MMU_PAGESHIFT + "
2856  	"; "
2857  
2858  	": TTE_IS_VALID ( ttep -- flag ) "
2859  	"   PHYS-X@ 0< "
2860  	"; "
2861  
2862  	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2863  	"   dup TTE8K =  if "
2864  	"      drop HBLK_RANGE_SHIFT "
2865  	"   else "
2866  	"      TTE_PAGE_SHIFT "
2867  	"   then "
2868  	"; "
2869  
2870  	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2871  	"   tuck >> swap MMU_PAGESHIFT - << "
2872  	"; "
2873  
2874  	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2875  	"   >> over xor swap                    ( hash sfmmup ) "
2876  	"   KHATID <>  if                       ( hash ) "
2877  	"      UHMEHASH_SZ and                  ( bucket ) "
2878  	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2879  	"   else                                ( hash ) "
2880  	"      KHMEHASH_SZ and                  ( bucket ) "
2881  	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2882  	"   then                                ( hmebp ) "
2883  	"; "
2884  
2885  	": HME_HASH_TABLE_SEARCH "
2886  	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2887  	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2888  	"      dup HMEBLK_ENDPA <> if     ( sfmmup hmeblkp ) ( r: hblktag ) "
2889  	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2890  	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2891  	"		  true	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2892  	"	     else						  "
2893  	"		  hmeblk_next + phys-x@ false			  "
2894  	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2895  	"	     then						  "
2896  	"	  else							  "
2897  	"	     hmeblk_next + phys-x@ false			  "
2898  	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2899  	"	  then							  "
2900  	"      else							  "
2901  	"         drop 0 true						  "
2902  	"      then							  "
2903  	"   until r> drop						  "
2904  	"; "
2905  
2906  	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2907  	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2908  	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2909  	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2910  	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2911  	"; "
2912  
2913  	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2914  	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2915  	"   TTE8K =  if                            ( hmeblkp addr ) "
2916  	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2917  	"   else                                   ( hmeblkp addr ) "
2918  	"      drop 0                              ( hmeblkp 0 ) "
2919  	"   then                                   ( hmeblkp hme-index ) "
2920  	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2921  	"   SFHME_TTE +                            ( ttep ) "
2922  	"; "
2923  
2924  	": unix-tte ( addr cnum -- false | tte-data true ) "
2925  	"    KCONTEXT = if                   ( addr ) "
2926  	"	KHATID                       ( addr khatid ) "
2927  	"    else                            ( addr ) "
2928  	"       drop false exit              ( false ) "
2929  	"    then "
2930  	"      ( addr khatid ) "
2931  	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2932  	"         2dup swap i HME_HASH_SHIFT  "
2933  					"( addr sfmmup sfmmup addr hmeshift ) "
2934  	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2935  	"         over i 4 pick               "
2936  				"( addr sfmmup hmebp sfmmup rehash addr ) "
2937  	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2938  	"         HME_HASH_TABLE_SEARCH       "
2939  					"( addr sfmmup { null | hmeblkp } ) "
2940  	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2941  	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2942  	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2943  	"               PHYS-X@ true          ( tte-data true ) "
2944  	"            else                     ( invalid-tte ) "
2945  	"               drop false            ( false ) "
2946  	"            then                     ( false | tte-data true ) "
2947  	"            unloop exit              ( false | tte-data true ) "
2948  	"         then                        ( addr sfmmup ) "
2949  	"      loop                           ( addr sfmmup ) "
2950  	"      2drop false                    ( false ) "
2951  	"; "
2952  ;
2953  
2954  void
2955  create_va_to_tte(void)
2956  {
2957  	char *bp;
2958  	extern int khmehash_num, uhmehash_num;
2959  	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2960  
2961  #define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2962  
2963  	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2964  
2965  	/*
2966  	 * Teach obp how to parse our sw ttes.
2967  	 */
2968  	(void) sprintf(bp, obp_tte_str,
2969  	    MMU_PAGESHIFT,
2970  	    TTE8K,
2971  	    sizeof (struct sf_hment),
2972  	    OFFSET(struct sf_hment, hme_tte),
2973  	    OFFSET(struct hme_blk, hblk_tag),
2974  	    OFFSET(struct hme_blk, hblk_nextpa),
2975  	    OFFSET(struct hme_blk, hblk_misc),
2976  	    OFFSET(struct hme_blk, hblk_hme),
2977  	    NHMENTS,
2978  	    HBLK_SZMASK,
2979  	    HBLK_RANGE_SHIFT,
2980  	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2981  	    HMEBLK_ENDPA,
2982  	    sizeof (struct hmehash_bucket),
2983  	    HTAG_SFMMUPSZ,
2984  	    HTAG_BSPAGE_SHIFT,
2985  	    HTAG_REHASH_SHIFT,
2986  	    SFMMU_INVALID_SHMERID,
2987  	    mmu_hashcnt,
2988  	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
2989  	    (caddr_t)va_to_pa((caddr_t)khme_hash),
2990  	    UHMEHASH_SZ,
2991  	    KHMEHASH_SZ,
2992  	    KCONTEXT,
2993  	    KHATID,
2994  	    ASI_MEM);
2995  	prom_interpret(bp, 0, 0, 0, 0, 0);
2996  
2997  	kobj_free(bp, MMU_PAGESIZE);
2998  }
2999  
3000  void
3001  install_va_to_tte(void)
3002  {
3003  	/*
3004  	 * advise prom that it can use unix-tte
3005  	 */
3006  	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
3007  }
3008  
3009  /*
3010   * Here we add "device-type=console" for /os-io node, for currently
3011   * our kernel console output only supports displaying text and
3012   * performing cursor-positioning operations (through kernel framebuffer
3013   * driver) and it doesn't support other functionalities required for a
3014   * standard "display" device as specified in 1275 spec. The main missing
3015   * interface defined by the 1275 spec is "draw-logo".
3016   * also see the comments above prom_stdout_is_framebuffer().
3017   */
3018  static char *create_node =
3019  	"\" /\" find-device "
3020  	"new-device "
3021  	"\" os-io\" device-name "
3022  	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
3023  	": cb-r/w  ( adr,len method$ -- #read/#written ) "
3024  	"   2>r swap 2 2r> ['] $callback  catch  if "
3025  	"      2drop 3drop 0 "
3026  	"   then "
3027  	"; "
3028  	": read ( adr,len -- #read ) "
3029  	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
3030  	"       ( retN ... ret1 N ) "
3031  	"       ?dup  if "
3032  	"               swap >r 1-  0  ?do  drop  loop  r> "
3033  	"       else "
3034  	"               -2 "
3035  	"       then "
3036  	";    "
3037  	": write ( adr,len -- #written ) "
3038  	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
3039  	"       ( retN ... ret1 N ) "
3040  	"       ?dup  if "
3041  	"               swap >r 1-  0  ?do  drop  loop  r> "
3042  	"        else "
3043  	"               0 "
3044  	"       then "
3045  	"; "
3046  	": poll-tty ( -- ) ; "
3047  	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
3048  	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
3049  	": cb-give/take ( $method -- ) "
3050  	"       0 -rot ['] $callback catch  ?dup  if "
3051  	"               >r 2drop 2drop r> throw "
3052  	"       else "
3053  	"               0  ?do  drop  loop "
3054  	"       then "
3055  	"; "
3056  	": give ( -- )  \" exit-input\" cb-give/take ; "
3057  	": take ( -- )  \" enter-input\" cb-give/take ; "
3058  	": open ( -- ok? )  true ; "
3059  	": close ( -- ) ; "
3060  	"finish-device "
3061  	"device-end ";
3062  
3063  /*
3064   * Create the OBP input/output node (FCode serial driver).
3065   * It is needed for both USB console keyboard and for
3066   * the kernel terminal emulator.  It is too early to check for a
3067   * kernel console compatible framebuffer now, so we create this
3068   * so that we're ready if we need to enable kernel terminal emulation.
3069   *
3070   * When the USB software takes over the input device at the time
3071   * consconfig runs, OBP's stdin is redirected to this node.
3072   * Whenever the FORTH user interface is used after this switch,
3073   * the node will call back into the kernel for console input.
3074   * If a serial device such as ttya or a UART with a Type 5 keyboard
3075   * attached is used, OBP takes over the serial device when the system
3076   * goes to the debugger after the system is booted.  This sharing
3077   * of the relatively simple serial device is difficult but possible.
3078   * Sharing the USB host controller is impossible due its complexity.
3079   *
3080   * Similarly to USB keyboard input redirection, after consconfig_dacf
3081   * configures a kernel console framebuffer as the standard output
3082   * device, OBP's stdout is switched to to vector through the
3083   * /os-io node into the kernel terminal emulator.
3084   */
3085  static void
3086  startup_create_io_node(void)
3087  {
3088  	prom_interpret(create_node, 0, 0, 0, 0, 0);
3089  }
3090  
3091  
3092  static void
3093  do_prom_version_check(void)
3094  {
3095  	int i;
3096  	pnode_t node;
3097  	char buf[64];
3098  	static char drev[] = "Down-rev firmware detected%s\n"
3099  	    "\tPlease upgrade to the following minimum version:\n"
3100  	    "\t\t%s\n";
3101  
3102  	i = prom_version_check(buf, sizeof (buf), &node);
3103  
3104  	if (i == PROM_VER64_OK)
3105  		return;
3106  
3107  	if (i == PROM_VER64_UPGRADE) {
3108  		cmn_err(CE_WARN, drev, "", buf);
3109  
3110  #ifdef	DEBUG
3111  		prom_enter_mon();	/* Type 'go' to continue */
3112  		cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3113  		return;
3114  #else
3115  		halt(0);
3116  #endif
3117  	}
3118  
3119  	/*
3120  	 * The other possibility is that this is a server running
3121  	 * good firmware, but down-rev firmware was detected on at
3122  	 * least one other cpu board. We just complain if we see
3123  	 * that.
3124  	 */
3125  	cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3126  }
3127  
3128  
3129  /*
3130   * Must be defined in platform dependent code.
3131   */
3132  extern caddr_t modtext;
3133  extern size_t modtext_sz;
3134  extern caddr_t moddata;
3135  
3136  #define	HEAPTEXT_ARENA(addr)	\
3137  	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3138  	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3139  	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3140  
3141  #define	HEAPTEXT_OVERSIZED(addr)	\
3142  	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3143  
3144  #define	HEAPTEXT_IN_NUCLEUSDATA(addr) \
3145  	(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3146  	((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3147  
3148  vmem_t *texthole_source[HEAPTEXT_NARENAS];
3149  vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3150  kmutex_t texthole_lock;
3151  
3152  char kern_bootargs[OBP_MAXPATHLEN];
3153  char kern_bootfile[OBP_MAXPATHLEN];
3154  
3155  void
3156  kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3157  {
3158  	uintptr_t addr, limit;
3159  
3160  	addr = HEAPTEXT_BASE;
3161  	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3162  
3163  	/*
3164  	 * Before we initialize the text_arena, we want to punch holes in the
3165  	 * underlying heaptext_arena.  This guarantees that for any text
3166  	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3167  	 */
3168  	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3169  	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3170  		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3171  		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3172  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3173  	}
3174  
3175  	/*
3176  	 * Allocate one page at the oversize to break up the text region
3177  	 * from the oversized region.
3178  	 */
3179  	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3180  	    (void *)limit, (void *)(limit + PAGESIZE),
3181  	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3182  
3183  	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3184  	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3185  	    heaptext_arena, 0, VM_SLEEP);
3186  	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3187  	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3188  }
3189  
3190  caddr_t
3191  kobj_text_alloc(vmem_t *arena, size_t size)
3192  {
3193  	caddr_t rval, better;
3194  
3195  	/*
3196  	 * First, try a sleeping allocation.
3197  	 */
3198  	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3199  
3200  	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3201  		return (rval);
3202  
3203  	/*
3204  	 * We didn't get the area that we wanted.  We're going to try to do an
3205  	 * allocation with explicit constraints.
3206  	 */
3207  	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3208  	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3209  	    VM_NOSLEEP | VM_BESTFIT);
3210  
3211  	if (better != NULL) {
3212  		/*
3213  		 * That worked.  Free our first attempt and return.
3214  		 */
3215  		vmem_free(arena, rval, size);
3216  		return (better);
3217  	}
3218  
3219  	/*
3220  	 * That didn't work; we'll have to return our first attempt.
3221  	 */
3222  	return (rval);
3223  }
3224  
3225  caddr_t
3226  kobj_texthole_alloc(caddr_t addr, size_t size)
3227  {
3228  	int arena = HEAPTEXT_ARENA(addr);
3229  	char c[30];
3230  	uintptr_t base;
3231  
3232  	if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3233  		/*
3234  		 * If this is an oversized allocation or it is allocated in
3235  		 * the nucleus data page, there is no text hole available for
3236  		 * it; return NULL.
3237  		 */
3238  		return (NULL);
3239  	}
3240  
3241  	mutex_enter(&texthole_lock);
3242  
3243  	if (texthole_arena[arena] == NULL) {
3244  		ASSERT(texthole_source[arena] == NULL);
3245  
3246  		if (arena == 0) {
3247  			texthole_source[0] = vmem_create("module_text_holesrc",
3248  			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3249  			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3250  			    0, VM_SLEEP);
3251  		} else {
3252  			base = HEAPTEXT_BASE +
3253  			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3254  
3255  			(void) snprintf(c, sizeof (c),
3256  			    "heaptext_holesrc_%d", arena);
3257  
3258  			texthole_source[arena] = vmem_create(c, (void *)base,
3259  			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3260  			    0, VM_SLEEP);
3261  		}
3262  
3263  		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3264  
3265  		texthole_arena[arena] = vmem_create(c, NULL, 0,
3266  		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3267  		    texthole_source[arena], 0, VM_SLEEP);
3268  	}
3269  
3270  	mutex_exit(&texthole_lock);
3271  
3272  	ASSERT(texthole_arena[arena] != NULL);
3273  	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3274  	return (vmem_alloc(texthole_arena[arena], size,
3275  	    VM_BESTFIT | VM_NOSLEEP));
3276  }
3277  
3278  void
3279  kobj_texthole_free(caddr_t addr, size_t size)
3280  {
3281  	int arena = HEAPTEXT_ARENA(addr);
3282  
3283  	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3284  	ASSERT(texthole_arena[arena] != NULL);
3285  	vmem_free(texthole_arena[arena], addr, size);
3286  }
3287  
3288  void
3289  release_bootstrap(void)
3290  {
3291  	if (&cif_init)
3292  		cif_init();
3293  }
3294