xref: /titanic_50/usr/src/uts/sun4/os/startup.c (revision be6e67355fa2266ba427189118d91a9e67b9163f)
1  /*
2   * CDDL HEADER START
3   *
4   * The contents of this file are subject to the terms of the
5   * Common Development and Distribution License (the "License").
6   * You may not use this file except in compliance with the License.
7   *
8   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9   * or http://www.opensolaris.org/os/licensing.
10   * See the License for the specific language governing permissions
11   * and limitations under the License.
12   *
13   * When distributing Covered Code, include this CDDL HEADER in each
14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15   * If applicable, add the following below this CDDL HEADER, with the
16   * fields enclosed by brackets "[]" replaced with your own identifying
17   * information: Portions Copyright [yyyy] [name of copyright owner]
18   *
19   * CDDL HEADER END
20   */
21  
22  /*
23   * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24   * Use is subject to license terms.
25   */
26  
27  #include <sys/machsystm.h>
28  #include <sys/archsystm.h>
29  #include <sys/vm.h>
30  #include <sys/cpu.h>
31  #include <sys/atomic.h>
32  #include <sys/reboot.h>
33  #include <sys/kdi.h>
34  #include <sys/bootconf.h>
35  #include <sys/memlist_plat.h>
36  #include <sys/memlist_impl.h>
37  #include <sys/prom_plat.h>
38  #include <sys/prom_isa.h>
39  #include <sys/autoconf.h>
40  #include <sys/intreg.h>
41  #include <sys/ivintr.h>
42  #include <sys/fpu/fpusystm.h>
43  #include <sys/iommutsb.h>
44  #include <vm/vm_dep.h>
45  #include <vm/seg_dev.h>
46  #include <vm/seg_kmem.h>
47  #include <vm/seg_kpm.h>
48  #include <vm/seg_map.h>
49  #include <vm/seg_kp.h>
50  #include <sys/sysconf.h>
51  #include <vm/hat_sfmmu.h>
52  #include <sys/kobj.h>
53  #include <sys/sun4asi.h>
54  #include <sys/clconf.h>
55  #include <sys/platform_module.h>
56  #include <sys/panic.h>
57  #include <sys/cpu_sgnblk_defs.h>
58  #include <sys/clock.h>
59  #include <sys/cmn_err.h>
60  #include <sys/promif.h>
61  #include <sys/prom_debug.h>
62  #include <sys/traptrace.h>
63  #include <sys/memnode.h>
64  #include <sys/mem_cage.h>
65  #include <sys/mmu.h>
66  
67  extern void setup_trap_table(void);
68  extern int cpu_intrq_setup(struct cpu *);
69  extern void cpu_intrq_register(struct cpu *);
70  extern void contig_mem_init(void);
71  extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
72  extern void mach_dump_buffer_init(void);
73  extern void mach_descrip_init(void);
74  extern void mach_descrip_startup_fini(void);
75  extern void mach_memscrub(void);
76  extern void mach_fpras(void);
77  extern void mach_cpu_halt_idle(void);
78  extern void mach_hw_copy_limit(void);
79  extern void load_mach_drivers(void);
80  extern void load_tod_module(void);
81  #pragma weak load_tod_module
82  
83  extern int ndata_alloc_mmfsa(struct memlist *ndata);
84  #pragma weak ndata_alloc_mmfsa
85  
86  extern void cif_init(void);
87  #pragma weak cif_init
88  
89  extern void parse_idprom(void);
90  extern void add_vx_handler(char *, int, void (*)(cell_t *));
91  extern void mem_config_init(void);
92  extern void memseg_remap_init(void);
93  
94  extern void mach_kpm_init(void);
95  extern void pcf_init();
96  extern int size_pse_array(pgcnt_t, int);
97  
98  /*
99   * External Data:
100   */
101  extern int vac_size;	/* cache size in bytes */
102  extern uint_t vac_mask;	/* VAC alignment consistency mask */
103  extern uint_t vac_colors;
104  
105  /*
106   * Global Data Definitions:
107   */
108  
109  /*
110   * XXX - Don't port this to new architectures
111   * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
112   * 'romp' has no use with a prom with an IEEE 1275 client interface.
113   * The driver doesn't use the value, but it depends on the symbol.
114   */
115  void *romp;		/* veritas driver won't load without romp 4154976 */
116  /*
117   * Declare these as initialized data so we can patch them.
118   */
119  pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
120  pgcnt_t segkpsize =
121      btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
122  uint_t segmap_percent = 12; /* Size of segmap segment */
123  
124  int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
125  int vac_copyback = 1;
126  char *cache_mode = NULL;
127  int use_mix = 1;
128  int prom_debug = 0;
129  
130  caddr_t boot_tba;		/* %tba at boot - used by kmdb */
131  uint_t	tba_taken_over = 0;
132  
133  caddr_t s_text;			/* start of kernel text segment */
134  caddr_t e_text;			/* end of kernel text segment */
135  caddr_t s_data;			/* start of kernel data segment */
136  caddr_t e_data;			/* end of kernel data segment */
137  
138  caddr_t modtext;		/* beginning of module text */
139  size_t	modtext_sz;		/* size of module text */
140  caddr_t moddata;		/* beginning of module data reserve */
141  caddr_t e_moddata;		/* end of module data reserve */
142  
143  /*
144   * End of first block of contiguous kernel in 32-bit virtual address space
145   */
146  caddr_t		econtig32;	/* end of first blk of contiguous kernel */
147  
148  caddr_t		ncbase;		/* beginning of non-cached segment */
149  caddr_t		ncend;		/* end of non-cached segment */
150  
151  size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
152  caddr_t	nalloc_base;		/* beginning of nucleus allocation */
153  caddr_t nalloc_end;		/* end of nucleus allocatable memory */
154  caddr_t valloc_base;		/* beginning of kvalloc segment	*/
155  
156  caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
157  caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
158  size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
159  caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
160  int	kmem64_szc;		/* page size code */
161  uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
162  
163  uintptr_t shm_alignment;	/* VAC address consistency modulus */
164  struct memlist *phys_install;	/* Total installed physical memory */
165  struct memlist *phys_avail;	/* Available (unreserved) physical memory */
166  struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
167  struct memlist *nopp_list;	/* pages with no backing page structs */
168  struct memlist ndata;		/* memlist of nucleus allocatable memory */
169  int memexp_flag;		/* memory expansion card flag */
170  uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
171  pgcnt_t obp_pages;		/* Physical pages used by OBP */
172  
173  /*
174   * VM data structures
175   */
176  long page_hashsz;		/* Size of page hash table (power of two) */
177  struct page *pp_base;		/* Base of system page struct array */
178  size_t pp_sz;			/* Size in bytes of page struct array */
179  struct page **page_hash;	/* Page hash table */
180  pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
181  size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
182  int pse_shift;			/* log2(pse_table_size) */
183  struct seg ktextseg;		/* Segment used for kernel executable image */
184  struct seg kvalloc;		/* Segment used for "valloc" mapping */
185  struct seg kpseg;		/* Segment used for pageable kernel virt mem */
186  struct seg ktexthole;		/* Segment used for nucleus text hole */
187  struct seg kmapseg;		/* Segment used for generic kernel mappings */
188  struct seg kpmseg;		/* Segment used for physical mapping */
189  struct seg kdebugseg;		/* Segment used for the kernel debugger */
190  
191  void *kpm_pp_base;		/* Base of system kpm_page array */
192  size_t	kpm_pp_sz;		/* Size of system kpm_page array */
193  pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
194  
195  struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
196  struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
197  struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
198  
199  int segzio_fromheap = 0;	/* zio allocations occur from heap */
200  caddr_t segzio_base;		/* Base address of segzio */
201  pgcnt_t segziosize = 0;		/* size of zio segment in pages */
202  
203  /*
204   * debugger pages (if allocated)
205   */
206  struct vnode kdebugvp;
207  
208  /*
209   * VA range available to the debugger
210   */
211  const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
212  const size_t kdi_segdebugsize = SEGDEBUGSIZE;
213  
214  /*
215   * Segment for relocated kernel structures in 64-bit large RAM kernels
216   */
217  struct seg kmem64;
218  
219  struct memseg *memseg_free;
220  
221  struct vnode unused_pages_vp;
222  
223  /*
224   * VM data structures allocated early during boot.
225   */
226  size_t pagehash_sz;
227  uint64_t memlist_sz;
228  
229  char tbr_wr_addr_inited = 0;
230  
231  caddr_t	mpo_heap32_buf = NULL;
232  size_t	mpo_heap32_bufsz = 0;
233  
234  /*
235   * Static Routines:
236   */
237  static int ndata_alloc_memseg(struct memlist *, size_t);
238  static void memlist_new(uint64_t, uint64_t, struct memlist **);
239  static void memlist_add(uint64_t, uint64_t,
240  	struct memlist **, struct memlist **);
241  static void kphysm_init(void);
242  static void kvm_init(void);
243  static void install_kmem64_tte(void);
244  
245  static void startup_init(void);
246  static void startup_memlist(void);
247  static void startup_modules(void);
248  static void startup_bop_gone(void);
249  static void startup_vm(void);
250  static void startup_end(void);
251  static void setup_cage_params(void);
252  static void startup_create_io_node(void);
253  
254  static pgcnt_t npages;
255  static struct memlist *memlist;
256  void *memlist_end;
257  
258  static pgcnt_t bop_alloc_pages;
259  static caddr_t hblk_base;
260  uint_t hblk_alloc_dynamic = 0;
261  uint_t hblk1_min = H1MIN;
262  
263  
264  /*
265   * Hooks for unsupported platforms and down-rev firmware
266   */
267  int iam_positron(void);
268  #pragma weak iam_positron
269  static void do_prom_version_check(void);
270  
271  /*
272   * After receiving a thermal interrupt, this is the number of seconds
273   * to delay before shutting off the system, assuming
274   * shutdown fails.  Use /etc/system to change the delay if this isn't
275   * large enough.
276   */
277  int thermal_powerdown_delay = 1200;
278  
279  /*
280   * Used to hold off page relocations into the cage until OBP has completed
281   * its boot-time handoff of its resources to the kernel.
282   */
283  int page_relocate_ready = 0;
284  
285  /*
286   * Indicate if kmem64 allocation was done in small chunks
287   */
288  int kmem64_smchunks = 0;
289  
290  /*
291   * Enable some debugging messages concerning memory usage...
292   */
293  #ifdef  DEBUGGING_MEM
294  static int debugging_mem;
295  static void
296  printmemlist(char *title, struct memlist *list)
297  {
298  	if (!debugging_mem)
299  		return;
300  
301  	printf("%s\n", title);
302  
303  	while (list) {
304  		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
305  		    (uint32_t)(list->address >> 32), (uint32_t)list->address,
306  		    (uint32_t)(list->size >> 32), (uint32_t)(list->size));
307  		list = list->next;
308  	}
309  }
310  
311  void
312  printmemseg(struct memseg *memseg)
313  {
314  	if (!debugging_mem)
315  		return;
316  
317  	printf("memseg\n");
318  
319  	while (memseg) {
320  		prom_printf("\tpage = 0x%p, epage = 0x%p, "
321  		    "pfn = 0x%x, epfn = 0x%x\n",
322  		    memseg->pages, memseg->epages,
323  		    memseg->pages_base, memseg->pages_end);
324  		memseg = memseg->next;
325  	}
326  }
327  
328  #define	debug_pause(str)	halt((str))
329  #define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
330  #define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
331  #define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
332  #define	MPRINTF3(str, a, b, c) \
333  	if (debugging_mem) prom_printf((str), (a), (b), (c))
334  #else	/* DEBUGGING_MEM */
335  #define	MPRINTF(str)
336  #define	MPRINTF1(str, a)
337  #define	MPRINTF2(str, a, b)
338  #define	MPRINTF3(str, a, b, c)
339  #endif	/* DEBUGGING_MEM */
340  
341  
342  /*
343   *
344   *                    Kernel's Virtual Memory Layout.
345   *                       /-----------------------\
346   * 0xFFFFFFFF.FFFFFFFF  -|                       |-
347   *                       |   OBP's virtual page  |
348   *                       |        tables         |
349   * 0xFFFFFFFC.00000000  -|-----------------------|-
350   *                       :                       :
351   *                       :                       :
352   *                      -|-----------------------|-
353   *                       |       segzio          | (base and size vary)
354   * 0xFFFFFE00.00000000  -|-----------------------|-
355   *                       |                       |  Ultrasparc I/II support
356   *                       |    segkpm segment     |  up to 2TB of physical
357   *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
358   *                       |                       |
359   * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
360   *                       :                       :
361   *                       :                       :
362   * 0xFFFFF810.00000000  -|-----------------------|- hole_end
363   *                       |                       |      ^
364   *                       |  UltraSPARC I/II call |      |
365   *                       | bug requires an extra |      |
366   *                       | 4 GB of space between |      |
367   *                       |   hole and used RAM   |	|
368   *                       |                       |      |
369   * 0xFFFFF800.00000000  -|-----------------------|-     |
370   *                       |                       |      |
371   *                       | Virtual Address Hole  |   UltraSPARC
372   *                       |  on UltraSPARC I/II   |  I/II * ONLY *
373   *                       |                       |      |
374   * 0x00000800.00000000  -|-----------------------|-     |
375   *                       |                       |      |
376   *                       |  UltraSPARC I/II call |      |
377   *                       | bug requires an extra |      |
378   *                       | 4 GB of space between |      |
379   *                       |   hole and used RAM   |      |
380   *                       |                       |      v
381   * 0x000007FF.00000000  -|-----------------------|- hole_start -----
382   *                       :                       :		   ^
383   *                       :                       :		   |
384   *                       |-----------------------|                 |
385   *                       |                       |                 |
386   *                       |  ecache flush area    |                 |
387   *                       |  (twice largest e$)   |                 |
388   *                       |                       |                 |
389   * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
390   *                       | overmapped area       |   alignend_end  |
391   *                       | (kmem64_alignsize     |		   |
392   *                       |  boundary)            |		   |
393   * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
394   *                       |                       |		   |
395   *                       |   64-bit kernel ONLY  |		   |
396   *                       |                       |		   |
397   *                       |    kmem64 segment     |		   |
398   *                       |                       |		   |
399   *                       | (Relocated extra HME  |	     Approximately
400   *                       |   block allocations,  |	    1 TB of virtual
401   *                       |   memnode freelists,  |	     address space
402   *                       |    HME hash buckets,  |		   |
403   *                       | mml_table, kpmp_table,|		   |
404   *                       |  page_t array and     |		   |
405   *                       |  hashblock pool to    |		   |
406   *                       |   avoid hard-coded    |		   |
407   *                       |     32-bit vaddr      |		   |
408   *                       |     limitations)      |		   |
409   *                       |                       |		   v
410   * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
411   *                       |                       |
412   *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
413   *                       |                       |
414   * 0x00000300.00000000  -|-----------------------|- SYSBASE
415   *                       :                       :
416   *                       :                       :
417   *                      -|-----------------------|-
418   *                       |                       |
419   *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
420   *                       |                       |               256G MAX)
421   * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
422   *                       :                       :
423   *                       :                       :
424   *                      -|-----------------------|-
425   *                       |                       |
426   *                       |       segkp           |    SEGKPSIZE (2GB)
427   *                       |                       |
428   *                       |                       |
429   * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
430   *                       |                       |
431   * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
432   *                       |                       |       (SEGKPBASE - 0x400000)
433   * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
434   *                       |                       |       (MEMSCRUBBASE - NCARGS)
435   * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
436   *                       |                       |       (ARGSBASE - PPMAPSIZE)
437   * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
438   *                       |                       |
439   * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
440   *                       |                       |
441   * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
442   *                       :                       :
443   *                       :                       :
444   * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
445   *                       |                       |
446   *                       |         OBP           |
447   *                       |                       |
448   * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
449   *                       |         kmdb          |
450   * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
451   *                       :                       :
452   *                       :                       :
453   * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
454   *                       |                       |
455   *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
456   *                       |                       |    ~64MB)
457   * 0x00000000.70002000  -|-----------------------|
458   *                       |     panicbuf          |
459   * 0x00000000.70000000  -|-----------------------|- SYSBASE32
460   *                       |       boot-time       |
461   *                       |    temporary space    |
462   * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
463   *                       :                       :
464   *                       :                       :
465   *                       |                       |
466   *                       |-----------------------|- econtig32
467   *                       |    vm structures      |
468   * 0x00000000.01C00000   |-----------------------|- nalloc_end
469   *                       |         TSBs          |
470   *                       |-----------------------|- end/nalloc_base
471   *                       |   kernel data & bss   |
472   * 0x00000000.01800000  -|-----------------------|
473   *                       :   nucleus text hole   :
474   * 0x00000000.01400000  -|-----------------------|
475   *                       :                       :
476   *                       |-----------------------|
477   *                       |      module text      |
478   *                       |-----------------------|- e_text/modtext
479   *                       |      kernel text      |
480   *                       |-----------------------|
481   *                       |    trap table (48k)   |
482   * 0x00000000.01000000  -|-----------------------|- KERNELBASE
483   *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
484   *                       |-----------------------|
485   *                       |                       |
486   *                       |        invalid        |
487   *                       |                       |
488   * 0x00000000.00000000  _|_______________________|
489   *
490   *
491   *
492   *                   32-bit User Virtual Memory Layout.
493   *                       /-----------------------\
494   *                       |                       |
495   *                       |        invalid        |
496   *                       |                       |
497   *          0xFFC00000  -|-----------------------|- USERLIMIT
498   *                       |       user stack      |
499   *                       :                       :
500   *                       :                       :
501   *                       :                       :
502   *                       |       user data       |
503   *                      -|-----------------------|-
504   *                       |       user text       |
505   *          0x00002000  -|-----------------------|-
506   *                       |       invalid         |
507   *          0x00000000  _|_______________________|
508   *
509   *
510   *
511   *                   64-bit User Virtual Memory Layout.
512   *                       /-----------------------\
513   *                       |                       |
514   *                       |        invalid        |
515   *                       |                       |
516   *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
517   *                       |       user stack      |
518   *                       :                       :
519   *                       :                       :
520   *                       :                       :
521   *                       |       user data       |
522   *                      -|-----------------------|-
523   *                       |       user text       |
524   *  0x00000000.01000000 -|-----------------------|-
525   *                       |       invalid         |
526   *  0x00000000.00000000 _|_______________________|
527   */
528  
529  extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
530  extern uint64_t ecache_flush_address(void);
531  
532  #pragma weak load_platform_modules
533  #pragma weak plat_startup_memlist
534  #pragma weak ecache_init_scrub_flush_area
535  #pragma weak ecache_flush_address
536  
537  
538  /*
539   * By default the DR Cage is enabled for maximum OS
540   * MPSS performance.  Users needing to disable the cage mechanism
541   * can set this variable to zero via /etc/system.
542   * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
543   * will result in loss of DR functionality.
544   * Platforms wishing to disable kernel Cage by default
545   * should do so in their set_platform_defaults() routine.
546   */
547  int	kernel_cage_enable = 1;
548  
549  static void
550  setup_cage_params(void)
551  {
552  	void (*func)(void);
553  
554  	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
555  	if (func != NULL) {
556  		(*func)();
557  		return;
558  	}
559  
560  	if (kernel_cage_enable == 0) {
561  		return;
562  	}
563  	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
564  
565  	if (kcage_on) {
566  		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
567  	} else {
568  		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
569  	}
570  
571  }
572  
573  /*
574   * Machine-dependent startup code
575   */
576  void
577  startup(void)
578  {
579  	startup_init();
580  	if (&startup_platform)
581  		startup_platform();
582  	startup_memlist();
583  	startup_modules();
584  	setup_cage_params();
585  	startup_bop_gone();
586  	startup_vm();
587  	startup_end();
588  }
589  
590  struct regs sync_reg_buf;
591  uint64_t sync_tt;
592  
593  void
594  sync_handler(void)
595  {
596  	struct  panic_trap_info 	ti;
597  	int i;
598  
599  	/*
600  	 * Prevent trying to talk to the other CPUs since they are
601  	 * sitting in the prom and won't reply.
602  	 */
603  	for (i = 0; i < NCPU; i++) {
604  		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
605  			cpu[i]->cpu_flags &= ~CPU_READY;
606  			cpu[i]->cpu_flags |= CPU_QUIESCED;
607  			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
608  		}
609  	}
610  
611  	/*
612  	 * We've managed to get here without going through the
613  	 * normal panic code path. Try and save some useful
614  	 * information.
615  	 */
616  	if (!panicstr && (curthread->t_panic_trap == NULL)) {
617  		ti.trap_type = sync_tt;
618  		ti.trap_regs = &sync_reg_buf;
619  		ti.trap_addr = NULL;
620  		ti.trap_mmu_fsr = 0x0;
621  
622  		curthread->t_panic_trap = &ti;
623  	}
624  
625  	/*
626  	 * If we're re-entering the panic path, update the signature
627  	 * block so that the SC knows we're in the second part of panic.
628  	 */
629  	if (panicstr)
630  		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
631  
632  	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
633  	panic("sync initiated");
634  }
635  
636  
637  static void
638  startup_init(void)
639  {
640  	/*
641  	 * We want to save the registers while we're still in OBP
642  	 * so that we know they haven't been fiddled with since.
643  	 * (In principle, OBP can't change them just because it
644  	 * makes a callback, but we'd rather not depend on that
645  	 * behavior.)
646  	 */
647  	char		sync_str[] =
648  	    "warning @ warning off : sync "
649  	    "%%tl-c %%tstate h# %p x! "
650  	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
651  	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
652  	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
653  	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
654  	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
655  	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
656  	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
657  	    "sync ; warning !";
658  
659  	/*
660  	 * 20 == num of %p substrings
661  	 * 16 == max num of chars %p will expand to.
662  	 */
663  	char 		bp[sizeof (sync_str) + 16 * 20];
664  
665  	/*
666  	 * Initialize ptl1 stack for the 1st CPU.
667  	 */
668  	ptl1_init_cpu(&cpu0);
669  
670  	/*
671  	 * Initialize the address map for cache consistent mappings
672  	 * to random pages; must be done after vac_size is set.
673  	 */
674  	ppmapinit();
675  
676  	/*
677  	 * Initialize the PROM callback handler.
678  	 */
679  	init_vx_handler();
680  
681  	/*
682  	 * have prom call sync_callback() to handle the sync and
683  	 * save some useful information which will be stored in the
684  	 * core file later.
685  	 */
686  	(void) sprintf((char *)bp, sync_str,
687  	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
688  	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
689  	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
690  	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
691  	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
692  	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
693  	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
694  	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
695  	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
696  	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
697  	prom_interpret(bp, 0, 0, 0, 0, 0);
698  	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
699  }
700  
701  
702  size_t
703  calc_pp_sz(pgcnt_t npages)
704  {
705  
706  	return (npages * sizeof (struct page));
707  }
708  
709  size_t
710  calc_kpmpp_sz(pgcnt_t npages)
711  {
712  
713  	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
714  	kpm_pgsz = 1ull << kpm_pgshft;
715  	kpm_pgoff = kpm_pgsz - 1;
716  	kpmp2pshft = kpm_pgshft - PAGESHIFT;
717  	kpmpnpgs = 1 << kpmp2pshft;
718  
719  	if (kpm_smallpages == 0) {
720  		/*
721  		 * Avoid fragmentation problems in kphysm_init()
722  		 * by allocating for all of physical memory
723  		 */
724  		kpm_npages = ptokpmpr(physinstalled);
725  		return (kpm_npages * sizeof (kpm_page_t));
726  	} else {
727  		kpm_npages = npages;
728  		return (kpm_npages * sizeof (kpm_spage_t));
729  	}
730  }
731  
732  size_t
733  calc_pagehash_sz(pgcnt_t npages)
734  {
735  
736  	/*
737  	 * The page structure hash table size is a power of 2
738  	 * such that the average hash chain length is PAGE_HASHAVELEN.
739  	 */
740  	page_hashsz = npages / PAGE_HASHAVELEN;
741  	page_hashsz = 1 << highbit(page_hashsz);
742  	return (page_hashsz * sizeof (struct page *));
743  }
744  
745  int testkmem64_smchunks = 0;
746  
747  int
748  alloc_kmem64(caddr_t base, caddr_t end)
749  {
750  	int i;
751  	caddr_t aligned_end = NULL;
752  
753  	if (testkmem64_smchunks)
754  		return (1);
755  
756  	/*
757  	 * Make one large memory alloc after figuring out the 64-bit size. This
758  	 * will enable use of the largest page size appropriate for the system
759  	 * architecture.
760  	 */
761  	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
762  	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
763  	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
764  		size_t alloc_size, alignsize;
765  #if !defined(C_OBP)
766  		unsigned long long pa;
767  #endif	/* !C_OBP */
768  
769  		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
770  			continue;
771  		alignsize = TTEBYTES(i);
772  		kmem64_szc = i;
773  
774  		/* limit page size for small memory */
775  		if (mmu_btop(alignsize) > (npages >> 2))
776  			continue;
777  
778  		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
779  		alloc_size = aligned_end - base;
780  #if !defined(C_OBP)
781  		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
782  			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
783  				kmem64_pabase = pa;
784  				kmem64_aligned_end = aligned_end;
785  				install_kmem64_tte();
786  				break;
787  			} else {
788  				prom_free_phys(alloc_size, pa);
789  			}
790  		}
791  #else	/* !C_OBP */
792  		if (prom_alloc(base, alloc_size, alignsize) == base) {
793  			kmem64_pabase = va_to_pa(kmem64_base);
794  			kmem64_aligned_end = aligned_end;
795  			break;
796  		}
797  #endif	/* !C_OBP */
798  		if (i == TTE8K) {
799  #ifdef sun4v
800  			/* return failure to try small allocations */
801  			return (1);
802  #else
803  			prom_panic("kmem64 allocation failure");
804  #endif
805  		}
806  	}
807  	ASSERT(aligned_end != NULL);
808  	return (0);
809  }
810  
811  static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
812  static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
813  
814  #define	IVSIZE	roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \
815  			(MAX_RSVD_IV * sizeof (intr_vec_t)) + \
816  			(MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE)
817  
818  #if !defined(C_OBP)
819  /*
820   * Install a temporary tte handler in OBP for kmem64 area.
821   *
822   * We map kmem64 area with large pages before the trap table is taken
823   * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
824   * the same area. Duplicate tlb entries with different page sizes
825   * cause unpredicatble behavior.  To avoid this, we don't create
826   * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
827   * OBP).  Instead, we manage translations with a temporary va>tte-data
828   * handler (kmem64-tte).  This handler is replaced by unix-tte when
829   * the trap table is taken over.
830   *
831   * The temporary handler knows the physical address of the kmem64
832   * area. It uses the prom's pgmap@ Forth word for other addresses.
833   *
834   * We have to use BOP_ALLOC() method for C-OBP platforms because
835   * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
836   * sun4u platforms. On sun4u we flush tlb after trap table is taken
837   * over if we use large pages for kernel heap and kmem64. Since sun4u
838   * prom (unlike sun4v) calls va>tte-data first for client address
839   * translation prom's ttes for kmem64 can't get into TLB even if we
840   * later switch to prom's trap table again. C-OBP uses 4M pages for
841   * client mappings when possible so on all platforms we get the
842   * benefit from large mappings for kmem64 area immediately during
843   * boot.
844   *
845   * pseudo code:
846   * if (context != 0) {
847   * 	return false
848   * } else if (miss_va in range[kmem64_base, kmem64_end)) {
849   *	tte = tte_template +
850   *		(((miss_va & pagemask) - kmem64_base));
851   *	return tte, true
852   * } else {
853   *	return pgmap@ result
854   * }
855   */
856  char kmem64_obp_str[] =
857  	"h# %lx constant kmem64-base "
858  	"h# %lx constant kmem64-end "
859  	"h# %lx constant kmem64-pagemask "
860  	"h# %lx constant kmem64-template "
861  
862  	": kmem64-tte ( addr cnum -- false | tte-data true ) "
863  	"    if                                       ( addr ) "
864  	"       drop false exit then                  ( false ) "
865  	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
866  	"	kmem64-pagemask and                   ( addr' ) "
867  	"	kmem64-base -                         ( addr' ) "
868  	"	kmem64-template +                     ( tte ) "
869  	"	true                                  ( tte true ) "
870  	"    else                                     ( addr ) "
871  	"	pgmap@                                ( tte ) "
872  	"       dup 0< if true else drop false then   ( tte true  |  false ) "
873  	"    then                                     ( tte true  |  false ) "
874  	"; "
875  
876  	"' kmem64-tte is va>tte-data "
877  ;
878  
879  static void
880  install_kmem64_tte()
881  {
882  	char b[sizeof (kmem64_obp_str) + (4 * 16)];
883  	tte_t tte;
884  
885  	PRM_DEBUG(kmem64_pabase);
886  	PRM_DEBUG(kmem64_szc);
887  	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
888  	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
889  	PRM_DEBUG(tte.ll);
890  	(void) sprintf(b, kmem64_obp_str,
891  	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
892  	ASSERT(strlen(b) < sizeof (b));
893  	prom_interpret(b, 0, 0, 0, 0, 0);
894  }
895  #endif	/* !C_OBP */
896  
897  /*
898   * As OBP takes up some RAM when the system boots, pages will already be "lost"
899   * to the system and reflected in npages by the time we see it.
900   *
901   * We only want to allocate kernel structures in the 64-bit virtual address
902   * space on systems with enough RAM to make the overhead of keeping track of
903   * an extra kernel memory segment worthwhile.
904   *
905   * Since OBP has already performed its memory allocations by this point, if we
906   * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
907   * memory in the 64-bit virtual address space; otherwise keep allocations
908   * contiguous with we've mapped so far in the 32-bit virtual address space.
909   */
910  #define	MINMOVE_RAM_MB	((size_t)1900)
911  #define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
912  #define	BYTES_TO_MB(b) ((b) / 1048576ul)
913  
914  pgcnt_t	tune_npages = (pgcnt_t)
915  	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
916  
917  #pragma weak page_set_colorequiv_arr_cpu
918  extern void page_set_colorequiv_arr_cpu(void);
919  extern void page_set_colorequiv_arr(void);
920  
921  static pgcnt_t ramdisk_npages;
922  static struct memlist *old_phys_avail;
923  
924  kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
925  
926  static void
927  startup_memlist(void)
928  {
929  	size_t hmehash_sz, pagelist_sz, tt_sz;
930  	size_t psetable_sz;
931  	caddr_t alloc_base;
932  	caddr_t memspace;
933  	struct memlist *cur;
934  	size_t syslimit = (size_t)SYSLIMIT;
935  	size_t sysbase = (size_t)SYSBASE;
936  
937  	/*
938  	 * Initialize enough of the system to allow kmem_alloc to work by
939  	 * calling boot to allocate its memory until the time that
940  	 * kvm_init is completed.  The page structs are allocated after
941  	 * rounding up end to the nearest page boundary; the memsegs are
942  	 * initialized and the space they use comes from the kernel heap.
943  	 * With appropriate initialization, they can be reallocated later
944  	 * to a size appropriate for the machine's configuration.
945  	 *
946  	 * At this point, memory is allocated for things that will never
947  	 * need to be freed, this used to be "valloced".  This allows a
948  	 * savings as the pages don't need page structures to describe
949  	 * them because them will not be managed by the vm system.
950  	 */
951  
952  	/*
953  	 * We're loaded by boot with the following configuration (as
954  	 * specified in the sun4u/conf/Mapfile):
955  	 *
956  	 * 	text:		4 MB chunk aligned on a 4MB boundary
957  	 * 	data & bss:	4 MB chunk aligned on a 4MB boundary
958  	 *
959  	 * These two chunks will eventually be mapped by 2 locked 4MB
960  	 * ttes and will represent the nucleus of the kernel.  This gives
961  	 * us some free space that is already allocated, some or all of
962  	 * which is made available to kernel module text.
963  	 *
964  	 * The free space in the data-bss chunk is used for nucleus
965  	 * allocatable data structures and we reserve it using the
966  	 * nalloc_base and nalloc_end variables.  This space is currently
967  	 * being used for hat data structures required for tlb miss
968  	 * handling operations.  We align nalloc_base to a l2 cache
969  	 * linesize because this is the line size the hardware uses to
970  	 * maintain cache coherency.
971  	 * 512K is carved out for module data.
972  	 */
973  
974  	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
975  	e_moddata = moddata + MODDATA;
976  	nalloc_base = e_moddata;
977  
978  	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
979  	valloc_base = nalloc_base;
980  
981  	/*
982  	 * Calculate the start of the data segment.
983  	 */
984  	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
985  		prom_panic("nucleus data overflow");
986  
987  	PRM_DEBUG(moddata);
988  	PRM_DEBUG(nalloc_base);
989  	PRM_DEBUG(nalloc_end);
990  
991  	/*
992  	 * Remember any slop after e_text so we can give it to the modules.
993  	 */
994  	PRM_DEBUG(e_text);
995  	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
996  	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
997  		prom_panic("nucleus text overflow");
998  	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
999  	    modtext;
1000  	PRM_DEBUG(modtext);
1001  	PRM_DEBUG(modtext_sz);
1002  
1003  	init_boot_memlists();
1004  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1005  	    &boot_physavail, &boot_physavail_len,
1006  	    &boot_virtavail, &boot_virtavail_len);
1007  
1008  	/*
1009  	 * Remember what the physically available highest page is
1010  	 * so that dumpsys works properly, and find out how much
1011  	 * memory is installed.
1012  	 */
1013  	installed_top_size_memlist_array(boot_physinstalled,
1014  	    boot_physinstalled_len, &physmax, &physinstalled);
1015  	PRM_DEBUG(physinstalled);
1016  	PRM_DEBUG(physmax);
1017  
1018  	/* Fill out memory nodes config structure */
1019  	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1020  
1021  	/*
1022  	 * npages is the maximum of available physical memory possible.
1023  	 * (ie. it will never be more than this)
1024  	 *
1025  	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1026  	 * using phys_avail will underestimate what will end up being freed.
1027  	 * A better initial guess is just total memory minus the kernel text
1028  	 */
1029  	npages = physinstalled - btop(MMU_PAGESIZE4M);
1030  
1031  	/*
1032  	 * First allocate things that can go in the nucleus data page
1033  	 * (fault status, TSBs, dmv, CPUs)
1034  	 */
1035  	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1036  
1037  	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1038  		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1039  
1040  	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1041  		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1042  
1043  	if (ndata_alloc_dmv(&ndata) != 0)
1044  		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1045  
1046  	if (ndata_alloc_page_mutexs(&ndata) != 0)
1047  		cmn_err(CE_PANIC,
1048  		    "no more nucleus memory after page free lists alloc");
1049  
1050  	if (ndata_alloc_hat(&ndata, npages) != 0)
1051  		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1052  
1053  	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1054  		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1055  
1056  	/*
1057  	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1058  	 *
1059  	 * There are comments all over the SFMMU code warning of dire
1060  	 * consequences if the TSBs are moved out of 32-bit space.  This
1061  	 * is largely because the asm code uses "sethi %hi(addr)"-type
1062  	 * instructions which will not provide the expected result if the
1063  	 * address is a 64-bit one.
1064  	 *
1065  	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1066  	 */
1067  	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1068  	PRM_DEBUG(alloc_base);
1069  
1070  	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1071  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1072  	PRM_DEBUG(alloc_base);
1073  
1074  	/*
1075  	 * Allocate IOMMU TSB array.  We do this here so that the physical
1076  	 * memory gets deducted from the PROM's physical memory list.
1077  	 */
1078  	alloc_base = iommu_tsb_init(alloc_base);
1079  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1080  	PRM_DEBUG(alloc_base);
1081  
1082  	/*
1083  	 * Allow for an early allocation of physically contiguous memory.
1084  	 */
1085  	alloc_base = contig_mem_prealloc(alloc_base, npages);
1086  
1087  	/*
1088  	 * Platforms like Starcat and OPL need special structures assigned in
1089  	 * 32-bit virtual address space because their probing routines execute
1090  	 * FCode, and FCode can't handle 64-bit virtual addresses...
1091  	 */
1092  	if (&plat_startup_memlist) {
1093  		alloc_base = plat_startup_memlist(alloc_base);
1094  		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1095  		    ecache_alignsize);
1096  		PRM_DEBUG(alloc_base);
1097  	}
1098  
1099  	/*
1100  	 * Save off where the contiguous allocations to date have ended
1101  	 * in econtig32.
1102  	 */
1103  	econtig32 = alloc_base;
1104  	PRM_DEBUG(econtig32);
1105  	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1106  		cmn_err(CE_PANIC, "econtig32 too big");
1107  
1108  	pp_sz = calc_pp_sz(npages);
1109  	PRM_DEBUG(pp_sz);
1110  	if (kpm_enable) {
1111  		kpm_pp_sz = calc_kpmpp_sz(npages);
1112  		PRM_DEBUG(kpm_pp_sz);
1113  	}
1114  
1115  	hmehash_sz = calc_hmehash_sz(npages);
1116  	PRM_DEBUG(hmehash_sz);
1117  
1118  	pagehash_sz = calc_pagehash_sz(npages);
1119  	PRM_DEBUG(pagehash_sz);
1120  
1121  	pagelist_sz = calc_free_pagelist_sz();
1122  	PRM_DEBUG(pagelist_sz);
1123  
1124  #ifdef	TRAPTRACE
1125  	tt_sz = calc_traptrace_sz();
1126  	PRM_DEBUG(tt_sz);
1127  #else
1128  	tt_sz = 0;
1129  #endif	/* TRAPTRACE */
1130  
1131  	/*
1132  	 * Place the array that protects pp->p_selock in the kmem64 wad.
1133  	 */
1134  	pse_shift = size_pse_array(npages, max_ncpus);
1135  	PRM_DEBUG(pse_shift);
1136  	pse_table_size = 1 << pse_shift;
1137  	PRM_DEBUG(pse_table_size);
1138  	psetable_sz = roundup(
1139  	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1140  	PRM_DEBUG(psetable_sz);
1141  
1142  	/*
1143  	 * Now allocate the whole wad
1144  	 */
1145  	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1146  	    pagelist_sz + tt_sz + psetable_sz;
1147  	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1148  	kmem64_base = (caddr_t)syslimit;
1149  	kmem64_end = kmem64_base + kmem64_sz;
1150  	if (alloc_kmem64(kmem64_base, kmem64_end)) {
1151  		/*
1152  		 * Attempt for kmem64 to allocate one big
1153  		 * contiguous chunk of memory failed.
1154  		 * We get here because we are sun4v.
1155  		 * We will proceed by breaking up
1156  		 * the allocation into two attempts.
1157  		 * First, we allocate kpm_pp_sz, hmehash_sz,
1158  		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1159  		 * one contiguous chunk. This is a much smaller
1160  		 * chunk and we should get it, if not we panic.
1161  		 * Note that hmehash and tt need to be physically
1162  		 * (in the real address sense) contiguous.
1163  		 * Next, we use bop_alloc_chunk() to
1164  		 * to allocate the page_t structures.
1165  		 * This will allow the page_t to be allocated
1166  		 * in multiple smaller chunks.
1167  		 * In doing so, the assumption that page_t is
1168  		 * physically contiguous no longer hold, this is ok
1169  		 * for sun4v but not for sun4u.
1170  		 */
1171  		size_t  tmp_size;
1172  		caddr_t tmp_base;
1173  
1174  		pp_sz  = roundup(pp_sz, PAGESIZE);
1175  
1176  		/*
1177  		 * Allocate kpm_pp_sz, hmehash_sz,
1178  		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1179  		 */
1180  		tmp_base = kmem64_base + pp_sz;
1181  		tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1182  		    pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1183  		if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1184  			prom_panic("kmem64 prom_alloc contig failed");
1185  		PRM_DEBUG(tmp_base);
1186  		PRM_DEBUG(tmp_size);
1187  
1188  		/*
1189  		 * Allocate the page_ts
1190  		 */
1191  		if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1192  			prom_panic("kmem64 bop_alloc_chunk page_t failed");
1193  		PRM_DEBUG(kmem64_base);
1194  		PRM_DEBUG(pp_sz);
1195  
1196  		kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1197  		ASSERT(kmem64_aligned_end >= kmem64_end);
1198  
1199  		kmem64_smchunks = 1;
1200  	} else {
1201  
1202  		/*
1203  		 * We need to adjust pp_sz for the normal
1204  		 * case where kmem64 can allocate one large chunk
1205  		 */
1206  		if (kpm_smallpages == 0) {
1207  			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1208  		} else {
1209  			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1210  			    sizeof (kpm_spage_t));
1211  		}
1212  		pp_sz = npages * sizeof (struct page);
1213  	}
1214  
1215  	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1216  		cmn_err(CE_PANIC, "not enough kmem64 space");
1217  	PRM_DEBUG(kmem64_base);
1218  	PRM_DEBUG(kmem64_end);
1219  	PRM_DEBUG(kmem64_aligned_end);
1220  
1221  	/*
1222  	 * ... and divy it up
1223  	 */
1224  	alloc_base = kmem64_base;
1225  
1226  	pp_base = (page_t *)alloc_base;
1227  	alloc_base += pp_sz;
1228  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1229  	PRM_DEBUG(pp_base);
1230  	PRM_DEBUG(npages);
1231  
1232  	if (kpm_enable) {
1233  		kpm_pp_base = alloc_base;
1234  		if (kpm_smallpages == 0) {
1235  			/* kpm_npages based on physinstalled, don't reset */
1236  			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1237  		} else {
1238  			kpm_npages = ptokpmpr(npages);
1239  			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1240  		}
1241  		alloc_base += kpm_pp_sz;
1242  		alloc_base =
1243  		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1244  		PRM_DEBUG(kpm_pp_base);
1245  	}
1246  
1247  	alloc_base = alloc_hmehash(alloc_base);
1248  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1249  	PRM_DEBUG(alloc_base);
1250  
1251  	page_hash = (page_t **)alloc_base;
1252  	alloc_base += pagehash_sz;
1253  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1254  	PRM_DEBUG(page_hash);
1255  
1256  	alloc_base = alloc_page_freelists(alloc_base);
1257  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1258  	PRM_DEBUG(alloc_base);
1259  
1260  #ifdef	TRAPTRACE
1261  	ttrace_buf = alloc_base;
1262  	alloc_base += tt_sz;
1263  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1264  	PRM_DEBUG(alloc_base);
1265  #endif	/* TRAPTRACE */
1266  
1267  	pse_mutex = (pad_mutex_t *)alloc_base;
1268  	alloc_base += psetable_sz;
1269  	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1270  	PRM_DEBUG(alloc_base);
1271  
1272  	/*
1273  	 * Note that if we use small chunk allocations for
1274  	 * kmem64, we need to ensure kmem64_end is the same as
1275  	 * kmem64_aligned_end to prevent subsequent logic from
1276  	 * trying to reuse the overmapping.
1277  	 * Otherwise we adjust kmem64_end to what we really allocated.
1278  	 */
1279  	if (kmem64_smchunks) {
1280  		kmem64_end = kmem64_aligned_end;
1281  	} else {
1282  		kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1283  	}
1284  	kmem64_sz = kmem64_end - kmem64_base;
1285  
1286  	if (&ecache_init_scrub_flush_area) {
1287  		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1288  		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1289  	}
1290  
1291  	/*
1292  	 * If physmem is patched to be non-zero, use it instead of
1293  	 * the monitor value unless physmem is larger than the total
1294  	 * amount of memory on hand.
1295  	 */
1296  	if (physmem == 0 || physmem > npages)
1297  		physmem = npages;
1298  
1299  	/*
1300  	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1301  	 * is mounted as root. This memory is held down by OBP and unlike
1302  	 * the stub boot_archive is never released.
1303  	 *
1304  	 * In order to get things sized correctly on lower memory
1305  	 * machines (where the memory used by the ramdisk represents
1306  	 * a significant portion of memory), physmem is adjusted.
1307  	 *
1308  	 * This is done by subtracting the ramdisk_size which is set
1309  	 * to the size of the ramdisk (in Kb) in /etc/system at the
1310  	 * time the miniroot archive is constructed.
1311  	 */
1312  	if (root_is_ramdisk == B_TRUE) {
1313  		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1314  		physmem -= ramdisk_npages;
1315  	}
1316  
1317  	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1318  		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1319  
1320  	/*
1321  	 * Allocate space for the interrupt vector table.
1322  	 */
1323  	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1324  	if (memspace != (caddr_t)intr_vec_table)
1325  		prom_panic("interrupt vector table allocation failure");
1326  
1327  	/*
1328  	 * Between now and when we finish copying in the memory lists,
1329  	 * allocations happen so the space gets fragmented and the
1330  	 * lists longer.  Leave enough space for lists twice as
1331  	 * long as we have now; then roundup to a pagesize.
1332  	 */
1333  	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1334  	    prom_phys_avail_len() + prom_virt_avail_len());
1335  	memlist_sz *= 2;
1336  	memlist_sz = roundup(memlist_sz, PAGESIZE);
1337  	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1338  	if (memspace == NULL)
1339  		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1340  
1341  	memlist = (struct memlist *)memspace;
1342  	memlist_end = (char *)memspace + memlist_sz;
1343  	PRM_DEBUG(memlist);
1344  	PRM_DEBUG(memlist_end);
1345  
1346  	PRM_DEBUG(sysbase);
1347  	PRM_DEBUG(syslimit);
1348  	kernelheap_init((void *)sysbase, (void *)syslimit,
1349  	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1350  
1351  	/*
1352  	 * Take the most current snapshot we can by calling mem-update.
1353  	 */
1354  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1355  	    &boot_physavail, &boot_physavail_len,
1356  	    &boot_virtavail, &boot_virtavail_len);
1357  
1358  	/*
1359  	 * Remove the space used by prom_alloc from the kernel heap
1360  	 * plus the area actually used by the OBP (if any)
1361  	 * ignoring virtual addresses in virt_avail, above syslimit.
1362  	 */
1363  	virt_avail = memlist;
1364  	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1365  
1366  	for (cur = virt_avail; cur->next; cur = cur->next) {
1367  		uint64_t range_base, range_size;
1368  
1369  		if ((range_base = cur->address + cur->size) < (uint64_t)sysbase)
1370  			continue;
1371  		if (range_base >= (uint64_t)syslimit)
1372  			break;
1373  		/*
1374  		 * Limit the range to end at syslimit.
1375  		 */
1376  		range_size = MIN(cur->next->address,
1377  		    (uint64_t)syslimit) - range_base;
1378  		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1379  		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1380  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1381  	}
1382  
1383  	phys_avail = memlist;
1384  	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1385  
1386  	/*
1387  	 * Add any extra memory at the end of the ndata region if there's at
1388  	 * least a page to add.  There might be a few more pages available in
1389  	 * the middle of the ndata region, but for now they are ignored.
1390  	 */
1391  	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1392  	if (nalloc_base == NULL)
1393  		nalloc_base = nalloc_end;
1394  	ndata_remain_sz = nalloc_end - nalloc_base;
1395  
1396  	/*
1397  	 * Copy physinstalled list into kernel space.
1398  	 */
1399  	phys_install = memlist;
1400  	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1401  
1402  	/*
1403  	 * Create list of physical addrs we don't need pp's for:
1404  	 * kernel text 4M page
1405  	 * kernel data 4M page - ndata_remain_sz
1406  	 * kmem64 pages
1407  	 *
1408  	 * NB if adding any pages here, make sure no kpm page
1409  	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1410  	 */
1411  	nopp_list = memlist;
1412  	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1413  	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1414  	    &memlist, &nopp_list);
1415  
1416  	/* Don't add to nopp_list if kmem64 was allocated in smchunks */
1417  	if (!kmem64_smchunks)
1418  		memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1419  
1420  	if ((caddr_t)memlist > (memspace + memlist_sz))
1421  		prom_panic("memlist overflow");
1422  
1423  	/*
1424  	 * Size the pcf array based on the number of cpus in the box at
1425  	 * boot time.
1426  	 */
1427  	pcf_init();
1428  
1429  	/*
1430  	 * Initialize the page structures from the memory lists.
1431  	 */
1432  	kphysm_init();
1433  
1434  	availrmem_initial = availrmem = freemem;
1435  	PRM_DEBUG(availrmem);
1436  
1437  	/*
1438  	 * Some of the locks depend on page_hashsz being set!
1439  	 * kmem_init() depends on this; so, keep it here.
1440  	 */
1441  	page_lock_init();
1442  
1443  	/*
1444  	 * Initialize kernel memory allocator.
1445  	 */
1446  	kmem_init();
1447  
1448  	/*
1449  	 * Factor in colorequiv to check additional 'equivalent' bins
1450  	 */
1451  	if (&page_set_colorequiv_arr_cpu != NULL)
1452  		page_set_colorequiv_arr_cpu();
1453  	else
1454  		page_set_colorequiv_arr();
1455  
1456  	/*
1457  	 * Initialize bp_mapin().
1458  	 */
1459  	bp_init(shm_alignment, HAT_STRICTORDER);
1460  
1461  	/*
1462  	 * Reserve space for panicbuf, intr_vec_table, reserved interrupt
1463  	 * vector data structures and MPO mblock structs from the 32-bit heap.
1464  	 */
1465  	(void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
1466  	    panicbuf, panicbuf + PANICBUFSIZE,
1467  	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1468  
1469  	(void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
1470  	    intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
1471  	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1472  
1473  	if (mpo_heap32_bufsz > (size_t)0) {
1474  		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1475  		    PAGESIZE, 0, 0, mpo_heap32_buf,
1476  		    mpo_heap32_buf + mpo_heap32_bufsz,
1477  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1478  	}
1479  	mem_config_init();
1480  }
1481  
1482  static void
1483  startup_modules(void)
1484  {
1485  	int nhblk1, nhblk8;
1486  	size_t  nhblksz;
1487  	pgcnt_t pages_per_hblk;
1488  	size_t hme8blk_sz, hme1blk_sz;
1489  
1490  	/*
1491  	 * Let the platforms have a chance to change default
1492  	 * values before reading system file.
1493  	 */
1494  	if (&set_platform_defaults)
1495  		set_platform_defaults();
1496  
1497  	/*
1498  	 * Calculate default settings of system parameters based upon
1499  	 * maxusers, yet allow to be overridden via the /etc/system file.
1500  	 */
1501  	param_calc(0);
1502  
1503  	mod_setup();
1504  
1505  	/*
1506  	 * If this is a positron, complain and halt.
1507  	 */
1508  	if (&iam_positron && iam_positron()) {
1509  		cmn_err(CE_WARN, "This hardware platform is not supported"
1510  		    " by this release of Solaris.\n");
1511  #ifdef DEBUG
1512  		prom_enter_mon();	/* Type 'go' to resume */
1513  		cmn_err(CE_WARN, "Booting an unsupported platform.\n");
1514  		cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
1515  
1516  #else /* DEBUG */
1517  		halt(0);
1518  #endif /* DEBUG */
1519  	}
1520  
1521  	/*
1522  	 * If we are running firmware that isn't 64-bit ready
1523  	 * then complain and halt.
1524  	 */
1525  	do_prom_version_check();
1526  
1527  	/*
1528  	 * Initialize system parameters
1529  	 */
1530  	param_init();
1531  
1532  	/*
1533  	 * maxmem is the amount of physical memory we're playing with.
1534  	 */
1535  	maxmem = physmem;
1536  
1537  	/* Set segkp limits. */
1538  	ncbase = kdi_segdebugbase;
1539  	ncend = kdi_segdebugbase;
1540  
1541  	/*
1542  	 * Initialize the hat layer.
1543  	 */
1544  	hat_init();
1545  
1546  	/*
1547  	 * Initialize segment management stuff.
1548  	 */
1549  	seg_init();
1550  
1551  	/*
1552  	 * Create the va>tte handler, so the prom can understand
1553  	 * kernel translations.  The handler is installed later, just
1554  	 * as we are about to take over the trap table from the prom.
1555  	 */
1556  	create_va_to_tte();
1557  
1558  	/*
1559  	 * Load the forthdebugger (optional)
1560  	 */
1561  	forthdebug_init();
1562  
1563  	/*
1564  	 * Create OBP node for console input callbacks
1565  	 * if it is needed.
1566  	 */
1567  	startup_create_io_node();
1568  
1569  	if (modloadonly("fs", "specfs") == -1)
1570  		halt("Can't load specfs");
1571  
1572  	if (modloadonly("fs", "devfs") == -1)
1573  		halt("Can't load devfs");
1574  
1575  	if (modloadonly("misc", "swapgeneric") == -1)
1576  		halt("Can't load swapgeneric");
1577  
1578  	(void) modloadonly("sys", "lbl_edition");
1579  
1580  	dispinit();
1581  
1582  	/*
1583  	 * Infer meanings to the members of the idprom buffer.
1584  	 */
1585  	parse_idprom();
1586  
1587  	/* Read cluster configuration data. */
1588  	clconf_init();
1589  
1590  	setup_ddi();
1591  
1592  	/*
1593  	 * Lets take this opportunity to load the root device.
1594  	 */
1595  	if (loadrootmodules() != 0)
1596  		debug_enter("Can't load the root filesystem");
1597  
1598  	/*
1599  	 * Load tod driver module for the tod part found on this system.
1600  	 * Recompute the cpu frequency/delays based on tod as tod part
1601  	 * tends to keep time more accurately.
1602  	 */
1603  	if (&load_tod_module)
1604  		load_tod_module();
1605  
1606  	/*
1607  	 * Allow platforms to load modules which might
1608  	 * be needed after bootops are gone.
1609  	 */
1610  	if (&load_platform_modules)
1611  		load_platform_modules();
1612  
1613  	setcpudelay();
1614  
1615  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1616  	    &boot_physavail, &boot_physavail_len,
1617  	    &boot_virtavail, &boot_virtavail_len);
1618  
1619  	/*
1620  	 * Calculation and allocation of hmeblks needed to remap
1621  	 * the memory allocated by PROM till now.
1622  	 * Overestimate the number of hblk1 elements by assuming
1623  	 * worst case of TTE64K mappings.
1624  	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1625  	 */
1626  	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1627  	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1628  	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1629  	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1630  
1631  	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1632  
1633  	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1634  	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1635  	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1636  
1637  	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1638  
1639  	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1640  	nhblk8 = 0;
1641  	while (bop_alloc_pages > 1) {
1642  		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1643  		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1644  		bop_alloc_pages *= hme8blk_sz;
1645  		bop_alloc_pages = btopr(bop_alloc_pages);
1646  	}
1647  	nhblk8 += 2;
1648  
1649  	/*
1650  	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1651  	 * boundary, the number of hblk8's needed to map the entries in the
1652  	 * boot_virtavail list needs to be adjusted to take this into
1653  	 * consideration.  Thus, we need to add additional hblk8's since it
1654  	 * is possible that an hblk8 will not have all 8 slots used due to
1655  	 * alignment constraints.  Since there were boot_virtavail_len entries
1656  	 * in that list, we need to add that many hblk8's to the number
1657  	 * already calculated to make sure we don't underestimate.
1658  	 */
1659  	nhblk8 += boot_virtavail_len;
1660  	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1661  
1662  	/* Allocate in pagesize chunks */
1663  	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1664  	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1665  	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1666  }
1667  
1668  static void
1669  startup_bop_gone(void)
1670  {
1671  
1672  	/*
1673  	 * Destroy the MD initialized at startup
1674  	 * The startup initializes the MD framework
1675  	 * using prom and BOP alloc free it now.
1676  	 */
1677  	mach_descrip_startup_fini();
1678  
1679  	/*
1680  	 * We're done with prom allocations.
1681  	 */
1682  	bop_fini();
1683  
1684  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1685  	    &boot_physavail, &boot_physavail_len,
1686  	    &boot_virtavail, &boot_virtavail_len);
1687  
1688  	/*
1689  	 * setup physically contiguous area twice as large as the ecache.
1690  	 * this is used while doing displacement flush of ecaches
1691  	 */
1692  	if (&ecache_flush_address) {
1693  		ecache_flushaddr = ecache_flush_address();
1694  		if (ecache_flushaddr == (uint64_t)-1) {
1695  			cmn_err(CE_PANIC,
1696  			    "startup: no memory to set ecache_flushaddr");
1697  		}
1698  	}
1699  
1700  	/*
1701  	 * Virtual available next.
1702  	 */
1703  	ASSERT(virt_avail != NULL);
1704  	memlist_free_list(virt_avail);
1705  	virt_avail = memlist;
1706  	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1707  
1708  }
1709  
1710  
1711  /*
1712   * startup_fixup_physavail - called from mach_sfmmu.c after the final
1713   * allocations have been performed.  We can't call it in startup_bop_gone
1714   * since later operations can cause obp to allocate more memory.
1715   */
1716  void
1717  startup_fixup_physavail(void)
1718  {
1719  	struct memlist *cur;
1720  	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1721  
1722  	PRM_DEBUG(kmem64_overmap_size);
1723  
1724  	/*
1725  	 * take the most current snapshot we can by calling mem-update
1726  	 */
1727  	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1728  	    &boot_physavail, &boot_physavail_len,
1729  	    &boot_virtavail, &boot_virtavail_len);
1730  
1731  	/*
1732  	 * Copy phys_avail list, again.
1733  	 * Both the kernel/boot and the prom have been allocating
1734  	 * from the original list we copied earlier.
1735  	 */
1736  	cur = memlist;
1737  	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1738  
1739  	/*
1740  	 * Add any unused kmem64 memory from overmapped page
1741  	 * (Note: va_to_pa does not work for kmem64_end)
1742  	 */
1743  	if (kmem64_overmap_size) {
1744  		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1745  		    kmem64_overmap_size, &memlist, &cur);
1746  	}
1747  
1748  	/*
1749  	 * Add any extra memory after e_data we added to the phys_avail list
1750  	 * back to the old list.
1751  	 */
1752  	if (ndata_remain_sz >= MMU_PAGESIZE)
1753  		memlist_add(va_to_pa(nalloc_base),
1754  		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1755  
1756  	/*
1757  	 * There isn't any bounds checking on the memlist area
1758  	 * so ensure it hasn't overgrown.
1759  	 */
1760  	if ((caddr_t)memlist > (caddr_t)memlist_end)
1761  		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1762  
1763  	/*
1764  	 * The kernel removes the pages that were allocated for it from
1765  	 * the freelist, but we now have to find any -extra- pages that
1766  	 * the prom has allocated for it's own book-keeping, and remove
1767  	 * them from the freelist too. sigh.
1768  	 */
1769  	sync_memlists(phys_avail, cur);
1770  
1771  	ASSERT(phys_avail != NULL);
1772  
1773  	old_phys_avail = phys_avail;
1774  	phys_avail = cur;
1775  }
1776  
1777  void
1778  update_kcage_ranges(uint64_t addr, uint64_t len)
1779  {
1780  	pfn_t base = btop(addr);
1781  	pgcnt_t num = btop(len);
1782  	int rv;
1783  
1784  	rv = kcage_range_add(base, num, kcage_startup_dir);
1785  
1786  	if (rv == ENOMEM) {
1787  		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1788  		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1789  	} else if (rv != 0) {
1790  		/* catch this in debug kernels */
1791  		ASSERT(0);
1792  
1793  		cmn_err(CE_WARN, "unexpected kcage_range_add"
1794  		    " return value %d", rv);
1795  	}
1796  }
1797  
1798  static void
1799  startup_vm(void)
1800  {
1801  	size_t	i;
1802  	struct segmap_crargs a;
1803  	struct segkpm_crargs b;
1804  
1805  	uint64_t avmem;
1806  	caddr_t va;
1807  	pgcnt_t	max_phys_segkp;
1808  	int	mnode;
1809  
1810  	extern int use_brk_lpg, use_stk_lpg;
1811  
1812  	/*
1813  	 * get prom's mappings, create hments for them and switch
1814  	 * to the kernel context.
1815  	 */
1816  	hat_kern_setup();
1817  
1818  	/*
1819  	 * Take over trap table
1820  	 */
1821  	setup_trap_table();
1822  
1823  	/*
1824  	 * Install the va>tte handler, so that the prom can handle
1825  	 * misses and understand the kernel table layout in case
1826  	 * we need call into the prom.
1827  	 */
1828  	install_va_to_tte();
1829  
1830  	/*
1831  	 * Set a flag to indicate that the tba has been taken over.
1832  	 */
1833  	tba_taken_over = 1;
1834  
1835  	/* initialize MMU primary context register */
1836  	mmu_init_kcontext();
1837  
1838  	/*
1839  	 * The boot cpu can now take interrupts, x-calls, x-traps
1840  	 */
1841  	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1842  	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1843  
1844  	/*
1845  	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1846  	 */
1847  	tbr_wr_addr_inited = 1;
1848  
1849  	/*
1850  	 * Initialize VM system, and map kernel address space.
1851  	 */
1852  	kvm_init();
1853  
1854  	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1855  	if (kernel_cage_enable) {
1856  		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1857  	}
1858  	memlist_free_list(old_phys_avail);
1859  
1860  	/*
1861  	 * If the following is true, someone has patched
1862  	 * phsymem to be less than the number of pages that
1863  	 * the system actually has.  Remove pages until system
1864  	 * memory is limited to the requested amount.  Since we
1865  	 * have allocated page structures for all pages, we
1866  	 * correct the amount of memory we want to remove
1867  	 * by the size of the memory used to hold page structures
1868  	 * for the non-used pages.
1869  	 */
1870  	if (physmem + ramdisk_npages < npages) {
1871  		pgcnt_t diff, off;
1872  		struct page *pp;
1873  		struct seg kseg;
1874  
1875  		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1876  
1877  		off = 0;
1878  		diff = npages - (physmem + ramdisk_npages);
1879  		diff -= mmu_btopr(diff * sizeof (struct page));
1880  		kseg.s_as = &kas;
1881  		while (diff--) {
1882  			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1883  			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1884  			    &kseg, (caddr_t)off);
1885  			if (pp == NULL)
1886  				cmn_err(CE_PANIC, "limited physmem too much!");
1887  			page_io_unlock(pp);
1888  			page_downgrade(pp);
1889  			availrmem--;
1890  			off += MMU_PAGESIZE;
1891  		}
1892  	}
1893  
1894  	/*
1895  	 * When printing memory, show the total as physmem less
1896  	 * that stolen by a debugger.
1897  	 */
1898  	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1899  	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1900  	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1901  
1902  	avmem = (uint64_t)freemem << PAGESHIFT;
1903  	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1904  
1905  	/*
1906  	 * For small memory systems disable automatic large pages.
1907  	 */
1908  	if (physmem < privm_lpg_min_physmem) {
1909  		use_brk_lpg = 0;
1910  		use_stk_lpg = 0;
1911  	}
1912  
1913  	/*
1914  	 * Perform platform specific freelist processing
1915  	 */
1916  	if (&plat_freelist_process) {
1917  		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1918  			if (mem_node_config[mnode].exists)
1919  				plat_freelist_process(mnode);
1920  	}
1921  
1922  	/*
1923  	 * Initialize the segkp segment type.  We position it
1924  	 * after the configured tables and buffers (whose end
1925  	 * is given by econtig) and before V_WKBASE_ADDR.
1926  	 * Also in this area is segkmap (size SEGMAPSIZE).
1927  	 */
1928  
1929  	/* XXX - cache alignment? */
1930  	va = (caddr_t)SEGKPBASE;
1931  	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1932  
1933  	max_phys_segkp = (physmem * 2);
1934  
1935  	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1936  		segkpsize = btop(SEGKPDEFSIZE);
1937  		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1938  		    "segkpsize has been reset to %ld pages", segkpsize);
1939  	}
1940  
1941  	i = ptob(MIN(segkpsize, max_phys_segkp));
1942  
1943  	rw_enter(&kas.a_lock, RW_WRITER);
1944  	if (seg_attach(&kas, va, i, segkp) < 0)
1945  		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1946  	if (segkp_create(segkp) != 0)
1947  		cmn_err(CE_PANIC, "startup: segkp_create failed");
1948  	rw_exit(&kas.a_lock);
1949  
1950  	/*
1951  	 * kpm segment
1952  	 */
1953  	segmap_kpm = kpm_enable &&
1954  	    segmap_kpm && PAGESIZE == MAXBSIZE;
1955  
1956  	if (kpm_enable) {
1957  		rw_enter(&kas.a_lock, RW_WRITER);
1958  
1959  		/*
1960  		 * The segkpm virtual range range is larger than the
1961  		 * actual physical memory size and also covers gaps in
1962  		 * the physical address range for the following reasons:
1963  		 * . keep conversion between segkpm and physical addresses
1964  		 *   simple, cheap and unambiguous.
1965  		 * . avoid extension/shrink of the the segkpm in case of DR.
1966  		 * . avoid complexity for handling of virtual addressed
1967  		 *   caches, segkpm and the regular mapping scheme must be
1968  		 *   kept in sync wrt. the virtual color of mapped pages.
1969  		 * Any accesses to virtual segkpm ranges not backed by
1970  		 * physical memory will fall through the memseg pfn hash
1971  		 * and will be handled in segkpm_fault.
1972  		 * Additional kpm_size spaces needed for vac alias prevention.
1973  		 */
1974  		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1975  		    segkpm) < 0)
1976  			cmn_err(CE_PANIC, "cannot attach segkpm");
1977  
1978  		b.prot = PROT_READ | PROT_WRITE;
1979  		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1980  
1981  		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1982  			panic("segkpm_create segkpm");
1983  
1984  		rw_exit(&kas.a_lock);
1985  
1986  		mach_kpm_init();
1987  	}
1988  
1989  	if (!segzio_fromheap) {
1990  		size_t size;
1991  		size_t physmem_b = mmu_ptob(physmem);
1992  
1993  		/* size is in bytes, segziosize is in pages */
1994  		if (segziosize == 0) {
1995  			size = physmem_b;
1996  		} else {
1997  			size = mmu_ptob(segziosize);
1998  		}
1999  
2000  		if (size < SEGZIOMINSIZE) {
2001  			size = SEGZIOMINSIZE;
2002  		} else if (size > SEGZIOMAXSIZE) {
2003  			size = SEGZIOMAXSIZE;
2004  			/*
2005  			 * On 64-bit x86, we only have 2TB of KVA.  This exists
2006  			 * for parity with x86.
2007  			 *
2008  			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2009  			 * doesn't consume all of KVA.  However, if we have a
2010  			 * system that has more thant 512gb of physical memory,
2011  			 * we can actually consume about half of the difference
2012  			 * between 512gb and the rest of the available physical
2013  			 * memory.
2014  			 */
2015  			if (physmem_b > SEGZIOMAXSIZE) {
2016  				size += (physmem_b - SEGZIOMAXSIZE) / 2;
2017  		}
2018  		}
2019  		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2020  		/* put the base of the ZIO segment after the kpm segment */
2021  		segzio_base = kpm_vbase + (kpm_size * vac_colors);
2022  		PRM_DEBUG(segziosize);
2023  		PRM_DEBUG(segzio_base);
2024  
2025  		/*
2026  		 * On some platforms, kvm_init is called after the kpm
2027  		 * sizes have been determined.  On SPARC, kvm_init is called
2028  		 * before, so we have to attach the kzioseg after kvm is
2029  		 * initialized, otherwise we'll try to allocate from the boot
2030  		 * area since the kernel heap hasn't yet been configured.
2031  		 */
2032  		rw_enter(&kas.a_lock, RW_WRITER);
2033  
2034  		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2035  		    &kzioseg);
2036  		(void) segkmem_zio_create(&kzioseg);
2037  
2038  		/* create zio area covering new segment */
2039  		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2040  
2041  		rw_exit(&kas.a_lock);
2042  	}
2043  
2044  
2045  	/*
2046  	 * Now create generic mapping segment.  This mapping
2047  	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2048  	 * virtual address is greater than the amount of free
2049  	 * memory that is available, then we trim back the
2050  	 * segment size to that amount
2051  	 */
2052  	va = (caddr_t)SEGMAPBASE;
2053  
2054  	/*
2055  	 * 1201049: segkmap base address must be MAXBSIZE aligned
2056  	 */
2057  	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2058  
2059  	/*
2060  	 * Set size of segmap to percentage of freemem at boot,
2061  	 * but stay within the allowable range
2062  	 * Note we take percentage  before converting from pages
2063  	 * to bytes to avoid an overflow on 32-bit kernels.
2064  	 */
2065  	i = mmu_ptob((freemem * segmap_percent) / 100);
2066  
2067  	if (i < MINMAPSIZE)
2068  		i = MINMAPSIZE;
2069  
2070  	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2071  		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2072  
2073  	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
2074  
2075  	rw_enter(&kas.a_lock, RW_WRITER);
2076  	if (seg_attach(&kas, va, i, segkmap) < 0)
2077  		cmn_err(CE_PANIC, "cannot attach segkmap");
2078  
2079  	a.prot = PROT_READ | PROT_WRITE;
2080  	a.shmsize = shm_alignment;
2081  	a.nfreelist = 0;	/* use segmap driver defaults */
2082  
2083  	if (segmap_create(segkmap, (caddr_t)&a) != 0)
2084  		panic("segmap_create segkmap");
2085  	rw_exit(&kas.a_lock);
2086  
2087  	segdev_init();
2088  }
2089  
2090  static void
2091  startup_end(void)
2092  {
2093  	if ((caddr_t)memlist > (caddr_t)memlist_end)
2094  		panic("memlist overflow 2");
2095  	memlist_free_block((caddr_t)memlist,
2096  	    ((caddr_t)memlist_end - (caddr_t)memlist));
2097  	memlist = NULL;
2098  
2099  	/* enable page_relocation since OBP is now done */
2100  	page_relocate_ready = 1;
2101  
2102  	/*
2103  	 * Perform tasks that get done after most of the VM
2104  	 * initialization has been done but before the clock
2105  	 * and other devices get started.
2106  	 */
2107  	kern_setup1();
2108  
2109  	/*
2110  	 * Intialize the VM arenas for allocating physically
2111  	 * contiguus memory chunk for interrupt queues snd
2112  	 * allocate/register boot cpu's queues, if any and
2113  	 * allocate dump buffer for sun4v systems to store
2114  	 * extra crash information during crash dump
2115  	 */
2116  	contig_mem_init();
2117  	mach_descrip_init();
2118  
2119  	if (cpu_intrq_setup(CPU)) {
2120  		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2121  	}
2122  	cpu_intrq_register(CPU);
2123  	mach_htraptrace_setup(CPU->cpu_id);
2124  	mach_htraptrace_configure(CPU->cpu_id);
2125  	mach_dump_buffer_init();
2126  
2127  	/*
2128  	 * Initialize interrupt related stuff
2129  	 */
2130  	cpu_intr_alloc(CPU, NINTR_THREADS);
2131  
2132  	(void) splzs();			/* allow hi clock ints but not zs */
2133  
2134  	/*
2135  	 * Initialize errors.
2136  	 */
2137  	error_init();
2138  
2139  	/*
2140  	 * Note that we may have already used kernel bcopy before this
2141  	 * point - but if you really care about this, adb the use_hw_*
2142  	 * variables to 0 before rebooting.
2143  	 */
2144  	mach_hw_copy_limit();
2145  
2146  	/*
2147  	 * Install the "real" preemption guards before DDI services
2148  	 * are available.
2149  	 */
2150  	(void) prom_set_preprom(kern_preprom);
2151  	(void) prom_set_postprom(kern_postprom);
2152  	CPU->cpu_m.mutex_ready = 1;
2153  
2154  	/*
2155  	 * Initialize segnf (kernel support for non-faulting loads).
2156  	 */
2157  	segnf_init();
2158  
2159  	/*
2160  	 * Configure the root devinfo node.
2161  	 */
2162  	configure();		/* set up devices */
2163  	mach_cpu_halt_idle();
2164  }
2165  
2166  
2167  void
2168  post_startup(void)
2169  {
2170  #ifdef	PTL1_PANIC_DEBUG
2171  	extern void init_ptl1_thread(void);
2172  #endif	/* PTL1_PANIC_DEBUG */
2173  	extern void abort_sequence_init(void);
2174  
2175  	/*
2176  	 * Set the system wide, processor-specific flags to be passed
2177  	 * to userland via the aux vector for performance hints and
2178  	 * instruction set extensions.
2179  	 */
2180  	bind_hwcap();
2181  
2182  	/*
2183  	 * Startup memory scrubber (if any)
2184  	 */
2185  	mach_memscrub();
2186  
2187  	/*
2188  	 * Allocate soft interrupt to handle abort sequence.
2189  	 */
2190  	abort_sequence_init();
2191  
2192  	/*
2193  	 * Configure the rest of the system.
2194  	 * Perform forceloading tasks for /etc/system.
2195  	 */
2196  	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2197  	/*
2198  	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2199  	 * ON4.0: This must be fixed or restated in /etc/systems.
2200  	 */
2201  	(void) modload("fs", "procfs");
2202  
2203  	/* load machine class specific drivers */
2204  	load_mach_drivers();
2205  
2206  	/* load platform specific drivers */
2207  	if (&load_platform_drivers)
2208  		load_platform_drivers();
2209  
2210  	/* load vis simulation module, if we are running w/fpu off */
2211  	if (!fpu_exists) {
2212  		if (modload("misc", "vis") == -1)
2213  			halt("Can't load vis");
2214  	}
2215  
2216  	mach_fpras();
2217  
2218  	maxmem = freemem;
2219  
2220  #ifdef	PTL1_PANIC_DEBUG
2221  	init_ptl1_thread();
2222  #endif	/* PTL1_PANIC_DEBUG */
2223  }
2224  
2225  #ifdef	PTL1_PANIC_DEBUG
2226  int		ptl1_panic_test = 0;
2227  int		ptl1_panic_xc_one_test = 0;
2228  int		ptl1_panic_xc_all_test = 0;
2229  int		ptl1_panic_xt_one_test = 0;
2230  int		ptl1_panic_xt_all_test = 0;
2231  kthread_id_t	ptl1_thread_p = NULL;
2232  kcondvar_t	ptl1_cv;
2233  kmutex_t	ptl1_mutex;
2234  int		ptl1_recurse_count_threshold = 0x40;
2235  int		ptl1_recurse_trap_threshold = 0x3d;
2236  extern void	ptl1_recurse(int, int);
2237  extern void	ptl1_panic_xt(int, int);
2238  
2239  /*
2240   * Called once per second by timeout() to wake up
2241   * the ptl1_panic thread to see if it should cause
2242   * a trap to the ptl1_panic() code.
2243   */
2244  /* ARGSUSED */
2245  static void
2246  ptl1_wakeup(void *arg)
2247  {
2248  	mutex_enter(&ptl1_mutex);
2249  	cv_signal(&ptl1_cv);
2250  	mutex_exit(&ptl1_mutex);
2251  }
2252  
2253  /*
2254   * ptl1_panic cross call function:
2255   *     Needed because xc_one() and xc_some() can pass
2256   *	64 bit args but ptl1_recurse() expects ints.
2257   */
2258  static void
2259  ptl1_panic_xc(void)
2260  {
2261  	ptl1_recurse(ptl1_recurse_count_threshold,
2262  	    ptl1_recurse_trap_threshold);
2263  }
2264  
2265  /*
2266   * The ptl1 thread waits for a global flag to be set
2267   * and uses the recurse thresholds to set the stack depth
2268   * to cause a ptl1_panic() directly via a call to ptl1_recurse
2269   * or indirectly via the cross call and cross trap functions.
2270   *
2271   * This is useful testing stack overflows and normal
2272   * ptl1_panic() states with a know stack frame.
2273   *
2274   * ptl1_recurse() is an asm function in ptl1_panic.s that
2275   * sets the {In, Local, Out, and Global} registers to a
2276   * know state on the stack and just prior to causing a
2277   * test ptl1_panic trap.
2278   */
2279  static void
2280  ptl1_thread(void)
2281  {
2282  	mutex_enter(&ptl1_mutex);
2283  	while (ptl1_thread_p) {
2284  		cpuset_t	other_cpus;
2285  		int		cpu_id;
2286  		int		my_cpu_id;
2287  		int		target_cpu_id;
2288  		int		target_found;
2289  
2290  		if (ptl1_panic_test) {
2291  			ptl1_recurse(ptl1_recurse_count_threshold,
2292  			    ptl1_recurse_trap_threshold);
2293  		}
2294  
2295  		/*
2296  		 * Find potential targets for x-call and x-trap,
2297  		 * if any exist while preempt is disabled we
2298  		 * start a ptl1_panic if requested via a
2299  		 * globals.
2300  		 */
2301  		kpreempt_disable();
2302  		my_cpu_id = CPU->cpu_id;
2303  		other_cpus = cpu_ready_set;
2304  		CPUSET_DEL(other_cpus, CPU->cpu_id);
2305  		target_found = 0;
2306  		if (!CPUSET_ISNULL(other_cpus)) {
2307  			/*
2308  			 * Pick the first one
2309  			 */
2310  			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2311  				if (cpu_id == my_cpu_id)
2312  					continue;
2313  
2314  				if (CPU_XCALL_READY(cpu_id)) {
2315  					target_cpu_id = cpu_id;
2316  					target_found = 1;
2317  					break;
2318  				}
2319  			}
2320  			ASSERT(target_found);
2321  
2322  			if (ptl1_panic_xc_one_test) {
2323  				xc_one(target_cpu_id,
2324  				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2325  			}
2326  			if (ptl1_panic_xc_all_test) {
2327  				xc_some(other_cpus,
2328  				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2329  			}
2330  			if (ptl1_panic_xt_one_test) {
2331  				xt_one(target_cpu_id,
2332  				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2333  			}
2334  			if (ptl1_panic_xt_all_test) {
2335  				xt_some(other_cpus,
2336  				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2337  			}
2338  		}
2339  		kpreempt_enable();
2340  		(void) timeout(ptl1_wakeup, NULL, hz);
2341  		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2342  	}
2343  	mutex_exit(&ptl1_mutex);
2344  }
2345  
2346  /*
2347   * Called during early startup to create the ptl1_thread
2348   */
2349  void
2350  init_ptl1_thread(void)
2351  {
2352  	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2353  	    &p0, TS_RUN, 0);
2354  }
2355  #endif	/* PTL1_PANIC_DEBUG */
2356  
2357  
2358  static void
2359  memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2360  {
2361  	struct memlist *new;
2362  
2363  	new = *memlistp;
2364  	new->address = start;
2365  	new->size = len;
2366  	*memlistp = new + 1;
2367  }
2368  
2369  /*
2370   * Add to a memory list.
2371   * start = start of new memory segment
2372   * len = length of new memory segment in bytes
2373   * memlistp = pointer to array of available memory segment structures
2374   * curmemlistp = memory list to which to add segment.
2375   */
2376  static void
2377  memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2378  	struct memlist **curmemlistp)
2379  {
2380  	struct memlist *new = *memlistp;
2381  
2382  	memlist_new(start, len, memlistp);
2383  	memlist_insert(new, curmemlistp);
2384  }
2385  
2386  static int
2387  ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2388  {
2389  	int nseg;
2390  	size_t memseg_sz;
2391  	struct memseg *msp;
2392  
2393  	/*
2394  	 * The memseg list is for the chunks of physical memory that
2395  	 * will be managed by the vm system.  The number calculated is
2396  	 * a guess as boot may fragment it more when memory allocations
2397  	 * are made before kphysm_init().
2398  	 */
2399  	memseg_sz = (avail + 10) * sizeof (struct memseg);
2400  	memseg_sz = roundup(memseg_sz, PAGESIZE);
2401  	nseg = memseg_sz / sizeof (struct memseg);
2402  	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2403  	if (msp == NULL)
2404  		return (1);
2405  	PRM_DEBUG(memseg_free);
2406  
2407  	while (nseg--) {
2408  		msp->next = memseg_free;
2409  		memseg_free = msp;
2410  		msp++;
2411  	}
2412  	return (0);
2413  }
2414  
2415  /*
2416   * In the case of architectures that support dynamic addition of
2417   * memory at run-time there are two cases where memsegs need to
2418   * be initialized and added to the memseg list.
2419   * 1) memsegs that are constructed at startup.
2420   * 2) memsegs that are constructed at run-time on
2421   *    hot-plug capable architectures.
2422   * This code was originally part of the function kphysm_init().
2423   */
2424  
2425  static void
2426  memseg_list_add(struct memseg *memsegp)
2427  {
2428  	struct memseg **prev_memsegp;
2429  	pgcnt_t num;
2430  
2431  	/* insert in memseg list, decreasing number of pages order */
2432  
2433  	num = MSEG_NPAGES(memsegp);
2434  
2435  	for (prev_memsegp = &memsegs; *prev_memsegp;
2436  	    prev_memsegp = &((*prev_memsegp)->next)) {
2437  		if (num > MSEG_NPAGES(*prev_memsegp))
2438  			break;
2439  	}
2440  
2441  	memsegp->next = *prev_memsegp;
2442  	*prev_memsegp = memsegp;
2443  
2444  	if (kpm_enable) {
2445  		memsegp->nextpa = (memsegp->next) ?
2446  		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2447  
2448  		if (prev_memsegp != &memsegs) {
2449  			struct memseg *msp;
2450  			msp = (struct memseg *)((caddr_t)prev_memsegp -
2451  			    offsetof(struct memseg, next));
2452  			msp->nextpa = va_to_pa(memsegp);
2453  		} else {
2454  			memsegspa = va_to_pa(memsegs);
2455  		}
2456  	}
2457  }
2458  
2459  /*
2460   * PSM add_physmem_cb(). US-II and newer processors have some
2461   * flavor of the prefetch capability implemented. We exploit
2462   * this capability for optimum performance.
2463   */
2464  #define	PREFETCH_BYTES	64
2465  
2466  void
2467  add_physmem_cb(page_t *pp, pfn_t pnum)
2468  {
2469  	extern void	 prefetch_page_w(void *);
2470  
2471  	pp->p_pagenum = pnum;
2472  
2473  	/*
2474  	 * Prefetch one more page_t into E$. To prevent future
2475  	 * mishaps with the sizeof(page_t) changing on us, we
2476  	 * catch this on debug kernels if we can't bring in the
2477  	 * entire hpage with 2 PREFETCH_BYTES reads. See
2478  	 * also, sun4u/cpu/cpu_module.c
2479  	 */
2480  	/*LINTED*/
2481  	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2482  	prefetch_page_w((char *)pp);
2483  }
2484  
2485  /*
2486   * Find memseg with given pfn
2487   */
2488  static struct memseg *
2489  memseg_find(pfn_t base, pfn_t *next)
2490  {
2491  	struct memseg *seg;
2492  
2493  	if (next != NULL)
2494  		*next = LONG_MAX;
2495  	for (seg = memsegs; seg != NULL; seg = seg->next) {
2496  		if (base >= seg->pages_base && base < seg->pages_end)
2497  			return (seg);
2498  		if (next != NULL && seg->pages_base > base &&
2499  		    seg->pages_base < *next)
2500  			*next = seg->pages_base;
2501  	}
2502  	return (NULL);
2503  }
2504  
2505  extern struct vnode prom_ppages;
2506  
2507  /*
2508   * Put page allocated by OBP on prom_ppages
2509   */
2510  static void
2511  kphysm_erase(uint64_t addr, uint64_t len)
2512  {
2513  	struct page *pp;
2514  	struct memseg *seg;
2515  	pfn_t base = btop(addr), next;
2516  	pgcnt_t num = btop(len);
2517  
2518  	while (num != 0) {
2519  		pgcnt_t off, left;
2520  
2521  		seg = memseg_find(base, &next);
2522  		if (seg == NULL) {
2523  			if (next == LONG_MAX)
2524  				break;
2525  			left = MIN(next - base, num);
2526  			base += left, num -= left;
2527  			continue;
2528  		}
2529  		off = base - seg->pages_base;
2530  		pp = seg->pages + off;
2531  		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2532  		while (num != left) {
2533  			/*
2534  			 * init it, lock it, and hashin on prom_pages vp.
2535  			 *
2536  			 * XXX	vnode offsets on the prom_ppages vnode
2537  			 *	are page numbers (gack) for >32 bit
2538  			 *	physical memory machines.
2539  			 */
2540  			add_physmem_cb(pp, base);
2541  			if (page_trylock(pp, SE_EXCL) == 0)
2542  				cmn_err(CE_PANIC, "prom page locked");
2543  			(void) page_hashin(pp, &prom_ppages,
2544  			    (offset_t)base, NULL);
2545  			(void) page_pp_lock(pp, 0, 1);
2546  			pp++, base++, num--;
2547  		}
2548  	}
2549  }
2550  
2551  static page_t *ppnext;
2552  static pgcnt_t ppleft;
2553  
2554  static void *kpm_ppnext;
2555  static pgcnt_t kpm_ppleft;
2556  
2557  /*
2558   * Create a memseg
2559   */
2560  static void
2561  kphysm_memseg(uint64_t addr, uint64_t len)
2562  {
2563  	pfn_t base = btop(addr);
2564  	pgcnt_t num = btop(len);
2565  	struct memseg *seg;
2566  
2567  	seg = memseg_free;
2568  	memseg_free = seg->next;
2569  	ASSERT(seg != NULL);
2570  
2571  	seg->pages = ppnext;
2572  	seg->epages = ppnext + num;
2573  	seg->pages_base = base;
2574  	seg->pages_end = base + num;
2575  	ppnext += num;
2576  	ppleft -= num;
2577  
2578  	if (kpm_enable) {
2579  		pgcnt_t kpnum = ptokpmpr(num);
2580  
2581  		if (kpnum > kpm_ppleft)
2582  			panic("kphysm_memseg: kpm_pp overflow");
2583  		seg->pagespa = va_to_pa(seg->pages);
2584  		seg->epagespa = va_to_pa(seg->epages);
2585  		seg->kpm_pbase = kpmptop(ptokpmp(base));
2586  		seg->kpm_nkpmpgs = kpnum;
2587  		/*
2588  		 * In the kpm_smallpage case, the kpm array
2589  		 * is 1-1 wrt the page array
2590  		 */
2591  		if (kpm_smallpages) {
2592  			kpm_spage_t *kpm_pp = kpm_ppnext;
2593  
2594  			kpm_ppnext = kpm_pp + kpnum;
2595  			seg->kpm_spages = kpm_pp;
2596  			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2597  		} else {
2598  			kpm_page_t *kpm_pp = kpm_ppnext;
2599  
2600  			kpm_ppnext = kpm_pp + kpnum;
2601  			seg->kpm_pages = kpm_pp;
2602  			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2603  			/* ASSERT no kpm overlaps */
2604  			ASSERT(
2605  			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2606  			ASSERT(memseg_find(
2607  			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2608  		}
2609  		kpm_ppleft -= num;
2610  	}
2611  
2612  	memseg_list_add(seg);
2613  }
2614  
2615  /*
2616   * Add range to free list
2617   */
2618  void
2619  kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2620  {
2621  	struct page *pp;
2622  	struct memseg *seg;
2623  	pfn_t base = btop(addr);
2624  	pgcnt_t num = btop(len);
2625  
2626  	seg = memseg_find(base, NULL);
2627  	ASSERT(seg != NULL);
2628  	pp = seg->pages + (base - seg->pages_base);
2629  
2630  	if (reclaim) {
2631  		struct page *rpp = pp;
2632  		struct page *lpp = pp + num;
2633  
2634  		/*
2635  		 * page should be locked on prom_ppages
2636  		 * unhash and unlock it
2637  		 */
2638  		while (rpp < lpp) {
2639  			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages);
2640  			page_pp_unlock(rpp, 0, 1);
2641  			page_hashout(rpp, NULL);
2642  			page_unlock(rpp);
2643  			rpp++;
2644  		}
2645  	}
2646  
2647  	/*
2648  	 * add_physmem() initializes the PSM part of the page
2649  	 * struct by calling the PSM back with add_physmem_cb().
2650  	 * In addition it coalesces pages into larger pages as
2651  	 * it initializes them.
2652  	 */
2653  	add_physmem(pp, num, base);
2654  }
2655  
2656  /*
2657   * kphysm_init() tackles the problem of initializing physical memory.
2658   */
2659  static void
2660  kphysm_init(void)
2661  {
2662  	struct memlist *pmem;
2663  
2664  	ASSERT(page_hash != NULL && page_hashsz != 0);
2665  
2666  	ppnext = pp_base;
2667  	ppleft = npages;
2668  	kpm_ppnext = kpm_pp_base;
2669  	kpm_ppleft = kpm_npages;
2670  
2671  	/*
2672  	 * installed pages not on nopp_memlist go in memseg list
2673  	 */
2674  	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2675  
2676  	/*
2677  	 * Free the avail list
2678  	 */
2679  	for (pmem = phys_avail; pmem != NULL; pmem = pmem->next)
2680  		kphysm_add(pmem->address, pmem->size, 0);
2681  
2682  	/*
2683  	 * Erase pages that aren't available
2684  	 */
2685  	diff_memlists(phys_install, phys_avail, kphysm_erase);
2686  
2687  	build_pfn_hash();
2688  }
2689  
2690  /*
2691   * Kernel VM initialization.
2692   * Assumptions about kernel address space ordering:
2693   *	(1) gap (user space)
2694   *	(2) kernel text
2695   *	(3) kernel data/bss
2696   *	(4) gap
2697   *	(5) kernel data structures
2698   *	(6) gap
2699   *	(7) debugger (optional)
2700   *	(8) monitor
2701   *	(9) gap (possibly null)
2702   *	(10) dvma
2703   *	(11) devices
2704   */
2705  static void
2706  kvm_init(void)
2707  {
2708  	/*
2709  	 * Put the kernel segments in kernel address space.
2710  	 */
2711  	rw_enter(&kas.a_lock, RW_WRITER);
2712  	as_avlinit(&kas);
2713  
2714  	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2715  	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2716  	(void) segkmem_create(&ktextseg);
2717  
2718  	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2719  	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2720  	(void) segkmem_create(&ktexthole);
2721  
2722  	(void) seg_attach(&kas, (caddr_t)valloc_base,
2723  	    (size_t)(econtig32 - valloc_base), &kvalloc);
2724  	(void) segkmem_create(&kvalloc);
2725  
2726  	if (kmem64_base) {
2727  		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2728  		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2729  		(void) segkmem_create(&kmem64);
2730  	}
2731  
2732  	/*
2733  	 * We're about to map out /boot.  This is the beginning of the
2734  	 * system resource management transition. We can no longer
2735  	 * call into /boot for I/O or memory allocations.
2736  	 */
2737  	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2738  	(void) segkmem_create(&kvseg);
2739  	hblk_alloc_dynamic = 1;
2740  
2741  	/*
2742  	 * we need to preallocate pages for DR operations before enabling large
2743  	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2744  	 */
2745  	memseg_remap_init();
2746  
2747  	/* at this point we are ready to use large page heap */
2748  	segkmem_heap_lp_init();
2749  
2750  	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2751  	    &kvseg32);
2752  	(void) segkmem_create(&kvseg32);
2753  
2754  	/*
2755  	 * Create a segment for the debugger.
2756  	 */
2757  	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2758  	(void) segkmem_create(&kdebugseg);
2759  
2760  	rw_exit(&kas.a_lock);
2761  }
2762  
2763  char obp_tte_str[] =
2764  	"h# %x constant MMU_PAGESHIFT "
2765  	"h# %x constant TTE8K "
2766  	"h# %x constant SFHME_SIZE "
2767  	"h# %x constant SFHME_TTE "
2768  	"h# %x constant HMEBLK_TAG "
2769  	"h# %x constant HMEBLK_NEXT "
2770  	"h# %x constant HMEBLK_MISC "
2771  	"h# %x constant HMEBLK_HME1 "
2772  	"h# %x constant NHMENTS "
2773  	"h# %x constant HBLK_SZMASK "
2774  	"h# %x constant HBLK_RANGE_SHIFT "
2775  	"h# %x constant HMEBP_HBLK "
2776  	"h# %x constant HMEBUCKET_SIZE "
2777  	"h# %x constant HTAG_SFMMUPSZ "
2778  	"h# %x constant HTAG_BSPAGE_SHIFT "
2779  	"h# %x constant HTAG_REHASH_SHIFT "
2780  	"h# %x constant SFMMU_INVALID_SHMERID "
2781  	"h# %x constant mmu_hashcnt "
2782  	"h# %p constant uhme_hash "
2783  	"h# %p constant khme_hash "
2784  	"h# %x constant UHMEHASH_SZ "
2785  	"h# %x constant KHMEHASH_SZ "
2786  	"h# %p constant KCONTEXT "
2787  	"h# %p constant KHATID "
2788  	"h# %x constant ASI_MEM "
2789  
2790  	": PHYS-X@ ( phys -- data ) "
2791  	"   ASI_MEM spacex@ "
2792  	"; "
2793  
2794  	": PHYS-W@ ( phys -- data ) "
2795  	"   ASI_MEM spacew@ "
2796  	"; "
2797  
2798  	": PHYS-L@ ( phys -- data ) "
2799  	"   ASI_MEM spaceL@ "
2800  	"; "
2801  
2802  	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2803  	"   3 * MMU_PAGESHIFT + "
2804  	"; "
2805  
2806  	": TTE_IS_VALID ( ttep -- flag ) "
2807  	"   PHYS-X@ 0< "
2808  	"; "
2809  
2810  	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2811  	"   dup TTE8K =  if "
2812  	"      drop HBLK_RANGE_SHIFT "
2813  	"   else "
2814  	"      TTE_PAGE_SHIFT "
2815  	"   then "
2816  	"; "
2817  
2818  	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2819  	"   tuck >> swap MMU_PAGESHIFT - << "
2820  	"; "
2821  
2822  	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2823  	"   >> over xor swap                    ( hash sfmmup ) "
2824  	"   KHATID <>  if                       ( hash ) "
2825  	"      UHMEHASH_SZ and                  ( bucket ) "
2826  	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2827  	"   else                                ( hash ) "
2828  	"      KHMEHASH_SZ and                  ( bucket ) "
2829  	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2830  	"   then                                ( hmebp ) "
2831  	"; "
2832  
2833  	": HME_HASH_TABLE_SEARCH "
2834  	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2835  	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2836  	"      dup if   		( sfmmup hmeblkp ) ( r: hblktag ) "
2837  	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2838  	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2839  	"		  true 	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2840  	"	     else						  "
2841  	"	     	  hmeblk_next + phys-x@ false 			  "
2842  	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2843  	"	     then  						  "
2844  	"	  else							  "
2845  	"	     hmeblk_next + phys-x@ false 			  "
2846  	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2847  	"	  then 							  "
2848  	"      else							  "
2849  	"         true 							  "
2850  	"      then  							  "
2851  	"   until r> drop 						  "
2852  	"; "
2853  
2854  	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2855  	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2856  	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2857  	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2858  	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2859  	"; "
2860  
2861  	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2862  	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2863  	"   TTE8K =  if                            ( hmeblkp addr ) "
2864  	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2865  	"   else                                   ( hmeblkp addr ) "
2866  	"      drop 0                              ( hmeblkp 0 ) "
2867  	"   then                                   ( hmeblkp hme-index ) "
2868  	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2869  	"   SFHME_TTE +                            ( ttep ) "
2870  	"; "
2871  
2872  	": unix-tte ( addr cnum -- false | tte-data true ) "
2873  	"    KCONTEXT = if                   ( addr ) "
2874  	"	KHATID                       ( addr khatid ) "
2875  	"    else                            ( addr ) "
2876  	"       drop false exit              ( false ) "
2877  	"    then "
2878  	"      ( addr khatid ) "
2879  	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2880  	"         2dup swap i HME_HASH_SHIFT  "
2881  					"( addr sfmmup sfmmup addr hmeshift ) "
2882  	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2883  	"         over i 4 pick               "
2884  				"( addr sfmmup hmebp sfmmup rehash addr ) "
2885  	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2886  	"         HME_HASH_TABLE_SEARCH       "
2887  					"( addr sfmmup { null | hmeblkp } ) "
2888  	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2889  	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2890  	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2891  	"               PHYS-X@ true          ( tte-data true ) "
2892  	"            else                     ( invalid-tte ) "
2893  	"               drop false            ( false ) "
2894  	"            then                     ( false | tte-data true ) "
2895  	"            unloop exit              ( false | tte-data true ) "
2896  	"         then                        ( addr sfmmup ) "
2897  	"      loop                           ( addr sfmmup ) "
2898  	"      2drop false                    ( false ) "
2899  	"; "
2900  ;
2901  
2902  void
2903  create_va_to_tte(void)
2904  {
2905  	char *bp;
2906  	extern int khmehash_num, uhmehash_num;
2907  	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2908  
2909  #define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2910  
2911  	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2912  
2913  	/*
2914  	 * Teach obp how to parse our sw ttes.
2915  	 */
2916  	(void) sprintf(bp, obp_tte_str,
2917  	    MMU_PAGESHIFT,
2918  	    TTE8K,
2919  	    sizeof (struct sf_hment),
2920  	    OFFSET(struct sf_hment, hme_tte),
2921  	    OFFSET(struct hme_blk, hblk_tag),
2922  	    OFFSET(struct hme_blk, hblk_nextpa),
2923  	    OFFSET(struct hme_blk, hblk_misc),
2924  	    OFFSET(struct hme_blk, hblk_hme),
2925  	    NHMENTS,
2926  	    HBLK_SZMASK,
2927  	    HBLK_RANGE_SHIFT,
2928  	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2929  	    sizeof (struct hmehash_bucket),
2930  	    HTAG_SFMMUPSZ,
2931  	    HTAG_BSPAGE_SHIFT,
2932  	    HTAG_REHASH_SHIFT,
2933  	    SFMMU_INVALID_SHMERID,
2934  	    mmu_hashcnt,
2935  	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
2936  	    (caddr_t)va_to_pa((caddr_t)khme_hash),
2937  	    UHMEHASH_SZ,
2938  	    KHMEHASH_SZ,
2939  	    KCONTEXT,
2940  	    KHATID,
2941  	    ASI_MEM);
2942  	prom_interpret(bp, 0, 0, 0, 0, 0);
2943  
2944  	kobj_free(bp, MMU_PAGESIZE);
2945  }
2946  
2947  void
2948  install_va_to_tte(void)
2949  {
2950  	/*
2951  	 * advise prom that he can use unix-tte
2952  	 */
2953  	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
2954  }
2955  
2956  /*
2957   * Here we add "device-type=console" for /os-io node, for currently
2958   * our kernel console output only supports displaying text and
2959   * performing cursor-positioning operations (through kernel framebuffer
2960   * driver) and it doesn't support other functionalities required for a
2961   * standard "display" device as specified in 1275 spec. The main missing
2962   * interface defined by the 1275 spec is "draw-logo".
2963   * also see the comments above prom_stdout_is_framebuffer().
2964   */
2965  static char *create_node =
2966  	"\" /\" find-device "
2967  	"new-device "
2968  	"\" os-io\" device-name "
2969  	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
2970  	": cb-r/w  ( adr,len method$ -- #read/#written ) "
2971  	"   2>r swap 2 2r> ['] $callback  catch  if "
2972  	"      2drop 3drop 0 "
2973  	"   then "
2974  	"; "
2975  	": read ( adr,len -- #read ) "
2976  	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
2977  	"       ( retN ... ret1 N ) "
2978  	"       ?dup  if "
2979  	"               swap >r 1-  0  ?do  drop  loop  r> "
2980  	"       else "
2981  	"               -2 "
2982  	"       then "
2983  	";    "
2984  	": write ( adr,len -- #written ) "
2985  	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
2986  	"       ( retN ... ret1 N ) "
2987  	"       ?dup  if "
2988  	"               swap >r 1-  0  ?do  drop  loop  r> "
2989  	"        else "
2990  	"               0 "
2991  	"       then "
2992  	"; "
2993  	": poll-tty ( -- ) ; "
2994  	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
2995  	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
2996  	": cb-give/take ( $method -- ) "
2997  	"       0 -rot ['] $callback catch  ?dup  if "
2998  	"               >r 2drop 2drop r> throw "
2999  	"       else "
3000  	"               0  ?do  drop  loop "
3001  	"       then "
3002  	"; "
3003  	": give ( -- )  \" exit-input\" cb-give/take ; "
3004  	": take ( -- )  \" enter-input\" cb-give/take ; "
3005  	": open ( -- ok? )  true ; "
3006  	": close ( -- ) ; "
3007  	"finish-device "
3008  	"device-end ";
3009  
3010  /*
3011   * Create the OBP input/output node (FCode serial driver).
3012   * It is needed for both USB console keyboard and for
3013   * the kernel terminal emulator.  It is too early to check for a
3014   * kernel console compatible framebuffer now, so we create this
3015   * so that we're ready if we need to enable kernel terminal emulation.
3016   *
3017   * When the USB software takes over the input device at the time
3018   * consconfig runs, OBP's stdin is redirected to this node.
3019   * Whenever the FORTH user interface is used after this switch,
3020   * the node will call back into the kernel for console input.
3021   * If a serial device such as ttya or a UART with a Type 5 keyboard
3022   * attached is used, OBP takes over the serial device when the system
3023   * goes to the debugger after the system is booted.  This sharing
3024   * of the relatively simple serial device is difficult but possible.
3025   * Sharing the USB host controller is impossible due its complexity.
3026   *
3027   * Similarly to USB keyboard input redirection, after consconfig_dacf
3028   * configures a kernel console framebuffer as the standard output
3029   * device, OBP's stdout is switched to to vector through the
3030   * /os-io node into the kernel terminal emulator.
3031   */
3032  static void
3033  startup_create_io_node(void)
3034  {
3035  	prom_interpret(create_node, 0, 0, 0, 0, 0);
3036  }
3037  
3038  
3039  static void
3040  do_prom_version_check(void)
3041  {
3042  	int i;
3043  	pnode_t node;
3044  	char buf[64];
3045  	static char drev[] = "Down-rev firmware detected%s\n"
3046  	    "\tPlease upgrade to the following minimum version:\n"
3047  	    "\t\t%s\n";
3048  
3049  	i = prom_version_check(buf, sizeof (buf), &node);
3050  
3051  	if (i == PROM_VER64_OK)
3052  		return;
3053  
3054  	if (i == PROM_VER64_UPGRADE) {
3055  		cmn_err(CE_WARN, drev, "", buf);
3056  
3057  #ifdef	DEBUG
3058  		prom_enter_mon();	/* Type 'go' to continue */
3059  		cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3060  		return;
3061  #else
3062  		halt(0);
3063  #endif
3064  	}
3065  
3066  	/*
3067  	 * The other possibility is that this is a server running
3068  	 * good firmware, but down-rev firmware was detected on at
3069  	 * least one other cpu board. We just complain if we see
3070  	 * that.
3071  	 */
3072  	cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3073  }
3074  
3075  
3076  /*
3077   * Must be defined in platform dependent code.
3078   */
3079  extern caddr_t modtext;
3080  extern size_t modtext_sz;
3081  extern caddr_t moddata;
3082  
3083  #define	HEAPTEXT_ARENA(addr)	\
3084  	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3085  	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3086  	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3087  
3088  #define	HEAPTEXT_OVERSIZED(addr)	\
3089  	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3090  
3091  #define	HEAPTEXT_IN_NUCLEUSDATA(addr) \
3092  	(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3093  	((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3094  
3095  vmem_t *texthole_source[HEAPTEXT_NARENAS];
3096  vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3097  kmutex_t texthole_lock;
3098  
3099  char kern_bootargs[OBP_MAXPATHLEN];
3100  
3101  void
3102  kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3103  {
3104  	uintptr_t addr, limit;
3105  
3106  	addr = HEAPTEXT_BASE;
3107  	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3108  
3109  	/*
3110  	 * Before we initialize the text_arena, we want to punch holes in the
3111  	 * underlying heaptext_arena.  This guarantees that for any text
3112  	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3113  	 */
3114  	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3115  	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3116  		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3117  		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3118  		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3119  	}
3120  
3121  	/*
3122  	 * Allocate one page at the oversize to break up the text region
3123  	 * from the oversized region.
3124  	 */
3125  	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3126  	    (void *)limit, (void *)(limit + PAGESIZE),
3127  	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3128  
3129  	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3130  	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3131  	    heaptext_arena, 0, VM_SLEEP);
3132  	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3133  	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3134  }
3135  
3136  caddr_t
3137  kobj_text_alloc(vmem_t *arena, size_t size)
3138  {
3139  	caddr_t rval, better;
3140  
3141  	/*
3142  	 * First, try a sleeping allocation.
3143  	 */
3144  	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3145  
3146  	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3147  		return (rval);
3148  
3149  	/*
3150  	 * We didn't get the area that we wanted.  We're going to try to do an
3151  	 * allocation with explicit constraints.
3152  	 */
3153  	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3154  	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3155  	    VM_NOSLEEP | VM_BESTFIT);
3156  
3157  	if (better != NULL) {
3158  		/*
3159  		 * That worked.  Free our first attempt and return.
3160  		 */
3161  		vmem_free(arena, rval, size);
3162  		return (better);
3163  	}
3164  
3165  	/*
3166  	 * That didn't work; we'll have to return our first attempt.
3167  	 */
3168  	return (rval);
3169  }
3170  
3171  caddr_t
3172  kobj_texthole_alloc(caddr_t addr, size_t size)
3173  {
3174  	int arena = HEAPTEXT_ARENA(addr);
3175  	char c[30];
3176  	uintptr_t base;
3177  
3178  	if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3179  		/*
3180  		 * If this is an oversized allocation or it is allocated in
3181  		 * the nucleus data page, there is no text hole available for
3182  		 * it; return NULL.
3183  		 */
3184  		return (NULL);
3185  	}
3186  
3187  	mutex_enter(&texthole_lock);
3188  
3189  	if (texthole_arena[arena] == NULL) {
3190  		ASSERT(texthole_source[arena] == NULL);
3191  
3192  		if (arena == 0) {
3193  			texthole_source[0] = vmem_create("module_text_holesrc",
3194  			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3195  			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3196  			    0, VM_SLEEP);
3197  		} else {
3198  			base = HEAPTEXT_BASE +
3199  			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3200  
3201  			(void) snprintf(c, sizeof (c),
3202  			    "heaptext_holesrc_%d", arena);
3203  
3204  			texthole_source[arena] = vmem_create(c, (void *)base,
3205  			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3206  			    0, VM_SLEEP);
3207  		}
3208  
3209  		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3210  
3211  		texthole_arena[arena] = vmem_create(c, NULL, 0,
3212  		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3213  		    texthole_source[arena], 0, VM_SLEEP);
3214  	}
3215  
3216  	mutex_exit(&texthole_lock);
3217  
3218  	ASSERT(texthole_arena[arena] != NULL);
3219  	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3220  	return (vmem_alloc(texthole_arena[arena], size,
3221  	    VM_BESTFIT | VM_NOSLEEP));
3222  }
3223  
3224  void
3225  kobj_texthole_free(caddr_t addr, size_t size)
3226  {
3227  	int arena = HEAPTEXT_ARENA(addr);
3228  
3229  	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3230  	ASSERT(texthole_arena[arena] != NULL);
3231  	vmem_free(texthole_arena[arena], addr, size);
3232  }
3233  
3234  void
3235  release_bootstrap(void)
3236  {
3237  	if (&cif_init)
3238  		cif_init();
3239  }
3240