xref: /titanic_41/usr/src/uts/sun4/os/startup.c (revision 803376f094adaaf2e4d9aa1c1d0efd56b642d838)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <sys/machsystm.h>
27 #include <sys/archsystm.h>
28 #include <sys/vm.h>
29 #include <sys/cpu.h>
30 #include <sys/atomic.h>
31 #include <sys/reboot.h>
32 #include <sys/kdi.h>
33 #include <sys/bootconf.h>
34 #include <sys/memlist_plat.h>
35 #include <sys/memlist_impl.h>
36 #include <sys/prom_plat.h>
37 #include <sys/prom_isa.h>
38 #include <sys/autoconf.h>
39 #include <sys/ivintr.h>
40 #include <sys/fpu/fpusystm.h>
41 #include <sys/iommutsb.h>
42 #include <vm/vm_dep.h>
43 #include <vm/seg_dev.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/seg_kpm.h>
46 #include <vm/seg_map.h>
47 #include <vm/seg_kp.h>
48 #include <sys/sysconf.h>
49 #include <vm/hat_sfmmu.h>
50 #include <sys/kobj.h>
51 #include <sys/sun4asi.h>
52 #include <sys/clconf.h>
53 #include <sys/platform_module.h>
54 #include <sys/panic.h>
55 #include <sys/cpu_sgnblk_defs.h>
56 #include <sys/clock.h>
57 #include <sys/cmn_err.h>
58 #include <sys/dumphdr.h>
59 #include <sys/promif.h>
60 #include <sys/prom_debug.h>
61 #include <sys/traptrace.h>
62 #include <sys/memnode.h>
63 #include <sys/mem_cage.h>
64 #include <sys/mmu.h>
65 #include <sys/swap.h>
66 
67 extern void setup_trap_table(void);
68 extern int cpu_intrq_setup(struct cpu *);
69 extern void cpu_intrq_register(struct cpu *);
70 extern void contig_mem_init(void);
71 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
72 extern void mach_dump_buffer_init(void);
73 extern void mach_descrip_init(void);
74 extern void mach_descrip_startup_fini(void);
75 extern void mach_memscrub(void);
76 extern void mach_fpras(void);
77 extern void mach_cpu_halt_idle(void);
78 extern void mach_hw_copy_limit(void);
79 extern void load_mach_drivers(void);
80 extern void load_tod_module(void);
81 #pragma weak load_tod_module
82 
83 extern int ndata_alloc_mmfsa(struct memlist *ndata);
84 #pragma weak ndata_alloc_mmfsa
85 
86 extern void cif_init(void);
87 #pragma weak cif_init
88 
89 extern void parse_idprom(void);
90 extern void add_vx_handler(char *, int, void (*)(cell_t *));
91 extern void mem_config_init(void);
92 extern void memseg_remap_init(void);
93 
94 extern void mach_kpm_init(void);
95 extern void pcf_init();
96 extern int size_pse_array(pgcnt_t, int);
97 extern void pg_init();
98 
99 /*
100  * External Data:
101  */
102 extern int vac_size;	/* cache size in bytes */
103 extern uint_t vac_mask;	/* VAC alignment consistency mask */
104 extern uint_t vac_colors;
105 
106 /*
107  * Global Data Definitions:
108  */
109 
110 /*
111  * XXX - Don't port this to new architectures
112  * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
113  * 'romp' has no use with a prom with an IEEE 1275 client interface.
114  * The driver doesn't use the value, but it depends on the symbol.
115  */
116 void *romp;		/* veritas driver won't load without romp 4154976 */
117 /*
118  * Declare these as initialized data so we can patch them.
119  */
120 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
121 pgcnt_t segkpsize =
122     btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
123 uint_t segmap_percent = 6; /* Size of segmap segment */
124 
125 int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
126 int vac_copyback = 1;
127 char *cache_mode = NULL;
128 int use_mix = 1;
129 int prom_debug = 0;
130 
131 caddr_t boot_tba;		/* %tba at boot - used by kmdb */
132 uint_t	tba_taken_over = 0;
133 
134 caddr_t s_text;			/* start of kernel text segment */
135 caddr_t e_text;			/* end of kernel text segment */
136 caddr_t s_data;			/* start of kernel data segment */
137 caddr_t e_data;			/* end of kernel data segment */
138 
139 caddr_t modtext;		/* beginning of module text */
140 size_t	modtext_sz;		/* size of module text */
141 caddr_t moddata;		/* beginning of module data reserve */
142 caddr_t e_moddata;		/* end of module data reserve */
143 
144 /*
145  * End of first block of contiguous kernel in 32-bit virtual address space
146  */
147 caddr_t		econtig32;	/* end of first blk of contiguous kernel */
148 
149 caddr_t		ncbase;		/* beginning of non-cached segment */
150 caddr_t		ncend;		/* end of non-cached segment */
151 
152 size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
153 caddr_t	nalloc_base;		/* beginning of nucleus allocation */
154 caddr_t nalloc_end;		/* end of nucleus allocatable memory */
155 caddr_t valloc_base;		/* beginning of kvalloc segment	*/
156 
157 caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
158 caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
159 size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
160 caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
161 int	kmem64_szc;		/* page size code */
162 uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
163 
164 uintptr_t shm_alignment;	/* VAC address consistency modulus */
165 struct memlist *phys_install;	/* Total installed physical memory */
166 struct memlist *phys_avail;	/* Available (unreserved) physical memory */
167 struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
168 struct memlist *nopp_list;	/* pages with no backing page structs */
169 struct memlist ndata;		/* memlist of nucleus allocatable memory */
170 int memexp_flag;		/* memory expansion card flag */
171 uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
172 pgcnt_t obp_pages;		/* Physical pages used by OBP */
173 
174 /*
175  * VM data structures
176  */
177 long page_hashsz;		/* Size of page hash table (power of two) */
178 unsigned int page_hashsz_shift;	/* log2(page_hashsz) */
179 struct page *pp_base;		/* Base of system page struct array */
180 size_t pp_sz;			/* Size in bytes of page struct array */
181 struct page **page_hash;	/* Page hash table */
182 pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
183 size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
184 int pse_shift;			/* log2(pse_table_size) */
185 struct seg ktextseg;		/* Segment used for kernel executable image */
186 struct seg kvalloc;		/* Segment used for "valloc" mapping */
187 struct seg kpseg;		/* Segment used for pageable kernel virt mem */
188 struct seg ktexthole;		/* Segment used for nucleus text hole */
189 struct seg kmapseg;		/* Segment used for generic kernel mappings */
190 struct seg kpmseg;		/* Segment used for physical mapping */
191 struct seg kdebugseg;		/* Segment used for the kernel debugger */
192 
193 void *kpm_pp_base;		/* Base of system kpm_page array */
194 size_t	kpm_pp_sz;		/* Size of system kpm_page array */
195 pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
196 
197 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
198 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
199 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
200 
201 int segzio_fromheap = 0;	/* zio allocations occur from heap */
202 caddr_t segzio_base;		/* Base address of segzio */
203 pgcnt_t segziosize = 0;		/* size of zio segment in pages */
204 
205 /*
206  * A static DR page_t VA map is reserved that can map the page structures
207  * for a domain's entire RA space. The pages that backs this space are
208  * dynamically allocated and need not be physically contiguous.  The DR
209  * map size is derived from KPM size.
210  */
211 int ppvm_enable = 0;		/* Static virtual map for page structs */
212 page_t *ppvm_base;		/* Base of page struct map */
213 pgcnt_t ppvm_size = 0;		/* Size of page struct map */
214 
215 /*
216  * debugger pages (if allocated)
217  */
218 struct vnode kdebugvp;
219 
220 /*
221  * VA range available to the debugger
222  */
223 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
224 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
225 
226 /*
227  * Segment for relocated kernel structures in 64-bit large RAM kernels
228  */
229 struct seg kmem64;
230 
231 struct memseg *memseg_free;
232 
233 struct vnode unused_pages_vp;
234 
235 /*
236  * VM data structures allocated early during boot.
237  */
238 size_t pagehash_sz;
239 uint64_t memlist_sz;
240 
241 char tbr_wr_addr_inited = 0;
242 
243 caddr_t	mpo_heap32_buf = NULL;
244 size_t	mpo_heap32_bufsz = 0;
245 
246 /*
247  * Static Routines:
248  */
249 static int ndata_alloc_memseg(struct memlist *, size_t);
250 static void memlist_new(uint64_t, uint64_t, struct memlist **);
251 static void memlist_add(uint64_t, uint64_t,
252 	struct memlist **, struct memlist **);
253 static void kphysm_init(void);
254 static void kvm_init(void);
255 static void install_kmem64_tte(void);
256 
257 static void startup_init(void);
258 static void startup_memlist(void);
259 static void startup_modules(void);
260 static void startup_bop_gone(void);
261 static void startup_vm(void);
262 static void startup_end(void);
263 static void setup_cage_params(void);
264 static void startup_create_io_node(void);
265 
266 static pgcnt_t npages;
267 static struct memlist *memlist;
268 void *memlist_end;
269 
270 static pgcnt_t bop_alloc_pages;
271 static caddr_t hblk_base;
272 uint_t hblk_alloc_dynamic = 0;
273 uint_t hblk1_min = H1MIN;
274 
275 
276 /*
277  * Hooks for unsupported platforms and down-rev firmware
278  */
279 int iam_positron(void);
280 #pragma weak iam_positron
281 static void do_prom_version_check(void);
282 
283 /*
284  * After receiving a thermal interrupt, this is the number of seconds
285  * to delay before shutting off the system, assuming
286  * shutdown fails.  Use /etc/system to change the delay if this isn't
287  * large enough.
288  */
289 int thermal_powerdown_delay = 1200;
290 
291 /*
292  * Used to hold off page relocations into the cage until OBP has completed
293  * its boot-time handoff of its resources to the kernel.
294  */
295 int page_relocate_ready = 0;
296 
297 /*
298  * Indicate if kmem64 allocation was done in small chunks
299  */
300 int kmem64_smchunks = 0;
301 
302 /*
303  * Enable some debugging messages concerning memory usage...
304  */
305 #ifdef  DEBUGGING_MEM
306 static int debugging_mem;
307 static void
printmemlist(char * title,struct memlist * list)308 printmemlist(char *title, struct memlist *list)
309 {
310 	if (!debugging_mem)
311 		return;
312 
313 	printf("%s\n", title);
314 
315 	while (list) {
316 		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
317 		    (uint32_t)(list->ml_address >> 32),
318 		    (uint32_t)list->ml_address,
319 		    (uint32_t)(list->ml_size >> 32),
320 		    (uint32_t)(list->ml_size));
321 		list = list->ml_next;
322 	}
323 }
324 
325 void
printmemseg(struct memseg * memseg)326 printmemseg(struct memseg *memseg)
327 {
328 	if (!debugging_mem)
329 		return;
330 
331 	printf("memseg\n");
332 
333 	while (memseg) {
334 		prom_printf("\tpage = 0x%p, epage = 0x%p, "
335 		    "pfn = 0x%x, epfn = 0x%x\n",
336 		    memseg->pages, memseg->epages,
337 		    memseg->pages_base, memseg->pages_end);
338 		memseg = memseg->next;
339 	}
340 }
341 
342 #define	debug_pause(str)	halt((str))
343 #define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
344 #define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
345 #define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
346 #define	MPRINTF3(str, a, b, c) \
347 	if (debugging_mem) prom_printf((str), (a), (b), (c))
348 #else	/* DEBUGGING_MEM */
349 #define	MPRINTF(str)
350 #define	MPRINTF1(str, a)
351 #define	MPRINTF2(str, a, b)
352 #define	MPRINTF3(str, a, b, c)
353 #endif	/* DEBUGGING_MEM */
354 
355 
356 /*
357  *
358  *                    Kernel's Virtual Memory Layout.
359  *                       /-----------------------\
360  * 0xFFFFFFFF.FFFFFFFF  -|                       |-
361  *                       |   OBP's virtual page  |
362  *                       |        tables         |
363  * 0xFFFFFFFC.00000000  -|-----------------------|-
364  *                       :                       :
365  *                       :                       :
366  *                      -|-----------------------|-
367  *                       |       segzio          | (base and size vary)
368  * 0xFFFFFE00.00000000  -|-----------------------|-
369  *                       |                       |  Ultrasparc I/II support
370  *                       |    segkpm segment     |  up to 2TB of physical
371  *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
372  *                       |                       |
373  * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
374  *                       :                       :
375  *                       :                       :
376  * 0xFFFFF810.00000000  -|-----------------------|- hole_end
377  *                       |                       |      ^
378  *                       |  UltraSPARC I/II call |      |
379  *                       | bug requires an extra |      |
380  *                       | 4 GB of space between |      |
381  *                       |   hole and used RAM   |	|
382  *                       |                       |      |
383  * 0xFFFFF800.00000000  -|-----------------------|-     |
384  *                       |                       |      |
385  *                       | Virtual Address Hole  |   UltraSPARC
386  *                       |  on UltraSPARC I/II   |  I/II * ONLY *
387  *                       |                       |      |
388  * 0x00000800.00000000  -|-----------------------|-     |
389  *                       |                       |      |
390  *                       |  UltraSPARC I/II call |      |
391  *                       | bug requires an extra |      |
392  *                       | 4 GB of space between |      |
393  *                       |   hole and used RAM   |      |
394  *                       |                       |      v
395  * 0x000007FF.00000000  -|-----------------------|- hole_start -----
396  *                       :                       :		   ^
397  *                       :                       :		   |
398  *                       |-----------------------|                 |
399  *                       |                       |                 |
400  *                       |  ecache flush area    |                 |
401  *                       |  (twice largest e$)   |                 |
402  *                       |                       |                 |
403  * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
404  *                       | overmapped area       |   alignend_end  |
405  *                       | (kmem64_alignsize     |		   |
406  *                       |  boundary)            |		   |
407  * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
408  *                       |                       |		   |
409  *                       |   64-bit kernel ONLY  |		   |
410  *                       |                       |		   |
411  *                       |    kmem64 segment     |		   |
412  *                       |                       |		   |
413  *                       | (Relocated extra HME  |	     Approximately
414  *                       |   block allocations,  |	    1 TB of virtual
415  *                       |   memnode freelists,  |	     address space
416  *                       |    HME hash buckets,  |		   |
417  *                       | mml_table, kpmp_table,|		   |
418  *                       |  page_t array and     |		   |
419  *                       |  hashblock pool to    |		   |
420  *                       |   avoid hard-coded    |		   |
421  *                       |     32-bit vaddr      |		   |
422  *                       |     limitations)      |		   |
423  *                       |                       |		   v
424  * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
425  *                       |                       |
426  *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
427  *                       |                       |
428  * 0x00000300.00000000  -|-----------------------|- SYSBASE
429  *                       :                       :
430  *                       :                       :
431  *                      -|-----------------------|-
432  *                       |                       |
433  *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
434  *                       |                       |               256G MAX)
435  * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
436  *                       :                       :
437  *                       :                       :
438  *                      -|-----------------------|-
439  *                       |                       |
440  *                       |       segkp           |    SEGKPSIZE (2GB)
441  *                       |                       |
442  *                       |                       |
443  * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
444  *                       |                       |
445  * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
446  *                       |                       |       (SEGKPBASE - 0x400000)
447  * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
448  *                       |                       |       (MEMSCRUBBASE - NCARGS)
449  * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
450  *                       |                       |       (ARGSBASE - PPMAPSIZE)
451  * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
452  *                       |                       |
453  * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
454  *                       |                       |
455  * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
456  *                       :                       :
457  *                       :                       :
458  * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
459  *                       |                       |
460  *                       |         OBP           |
461  *                       |                       |
462  * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
463  *                       |         kmdb          |
464  * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
465  *                       :                       :
466  *                       :                       :
467  * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
468  *                       |                       |
469  *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
470  *                       |                       |    ~64MB)
471  *			-|-----------------------|
472  *			 |	IVSIZE		 |
473  * 0x00000000.70004000  -|-----------------------|
474  *                       |     panicbuf          |
475  * 0x00000000.70002000	-|-----------------------|
476  *			 |	PAGESIZE	 |
477  * 0x00000000.70000000  -|-----------------------|- SYSBASE32
478  *                       |       boot-time       |
479  *                       |    temporary space    |
480  * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
481  *                       :                       :
482  *                       :                       :
483  *                       |                       |
484  *                       |-----------------------|- econtig32
485  *                       |    vm structures      |
486  * 0x00000000.01C00000   |-----------------------|- nalloc_end
487  *                       |         TSBs          |
488  *                       |-----------------------|- end/nalloc_base
489  *                       |   kernel data & bss   |
490  * 0x00000000.01800000  -|-----------------------|
491  *                       :   nucleus text hole   :
492  * 0x00000000.01400000  -|-----------------------|
493  *                       :                       :
494  *                       |-----------------------|
495  *                       |      module text      |
496  *                       |-----------------------|- e_text/modtext
497  *                       |      kernel text      |
498  *                       |-----------------------|
499  *                       |    trap table (48k)   |
500  * 0x00000000.01000000  -|-----------------------|- KERNELBASE
501  *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
502  *                       |-----------------------|
503  *                       |                       |
504  *                       |        invalid        |
505  *                       |                       |
506  * 0x00000000.00000000  _|_______________________|
507  *
508  *
509  *
510  *                   32-bit User Virtual Memory Layout.
511  *                       /-----------------------\
512  *                       |                       |
513  *                       |        invalid        |
514  *                       |                       |
515  *          0xFFC00000  -|-----------------------|- USERLIMIT
516  *                       |       user stack      |
517  *                       :                       :
518  *                       :                       :
519  *                       :                       :
520  *                       |       user data       |
521  *                      -|-----------------------|-
522  *                       |       user text       |
523  *          0x00002000  -|-----------------------|-
524  *                       |       invalid         |
525  *          0x00000000  _|_______________________|
526  *
527  *
528  *
529  *                   64-bit User Virtual Memory Layout.
530  *                       /-----------------------\
531  *                       |                       |
532  *                       |        invalid        |
533  *                       |                       |
534  *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
535  *                       |       user stack      |
536  *                       :                       :
537  *                       :                       :
538  *                       :                       :
539  *                       |       user data       |
540  *                      -|-----------------------|-
541  *                       |       user text       |
542  *  0x00000000.01000000 -|-----------------------|-
543  *                       |       invalid         |
544  *  0x00000000.00000000 _|_______________________|
545  */
546 
547 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
548 extern uint64_t ecache_flush_address(void);
549 
550 #pragma weak load_platform_modules
551 #pragma weak plat_startup_memlist
552 #pragma weak ecache_init_scrub_flush_area
553 #pragma weak ecache_flush_address
554 
555 
556 /*
557  * By default the DR Cage is enabled for maximum OS
558  * MPSS performance.  Users needing to disable the cage mechanism
559  * can set this variable to zero via /etc/system.
560  * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
561  * will result in loss of DR functionality.
562  * Platforms wishing to disable kernel Cage by default
563  * should do so in their set_platform_defaults() routine.
564  */
565 int	kernel_cage_enable = 1;
566 
567 static void
setup_cage_params(void)568 setup_cage_params(void)
569 {
570 	void (*func)(void);
571 
572 	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
573 	if (func != NULL) {
574 		(*func)();
575 		return;
576 	}
577 
578 	if (kernel_cage_enable == 0) {
579 		return;
580 	}
581 	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
582 
583 	if (kcage_on) {
584 		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
585 	} else {
586 		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
587 	}
588 
589 }
590 
591 /*
592  * Machine-dependent startup code
593  */
594 void
startup(void)595 startup(void)
596 {
597 	startup_init();
598 	if (&startup_platform)
599 		startup_platform();
600 	startup_memlist();
601 	startup_modules();
602 	setup_cage_params();
603 	startup_bop_gone();
604 	startup_vm();
605 	startup_end();
606 }
607 
608 struct regs sync_reg_buf;
609 uint64_t sync_tt;
610 
611 void
sync_handler(void)612 sync_handler(void)
613 {
614 	struct  panic_trap_info 	ti;
615 	int i;
616 
617 	/*
618 	 * Prevent trying to talk to the other CPUs since they are
619 	 * sitting in the prom and won't reply.
620 	 */
621 	for (i = 0; i < NCPU; i++) {
622 		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
623 			cpu[i]->cpu_flags &= ~CPU_READY;
624 			cpu[i]->cpu_flags |= CPU_QUIESCED;
625 			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
626 		}
627 	}
628 
629 	/*
630 	 * Force a serial dump, since there are no CPUs to help.
631 	 */
632 	dump_plat_mincpu = 0;
633 
634 	/*
635 	 * We've managed to get here without going through the
636 	 * normal panic code path. Try and save some useful
637 	 * information.
638 	 */
639 	if (!panicstr && (curthread->t_panic_trap == NULL)) {
640 		ti.trap_type = sync_tt;
641 		ti.trap_regs = &sync_reg_buf;
642 		ti.trap_addr = NULL;
643 		ti.trap_mmu_fsr = 0x0;
644 
645 		curthread->t_panic_trap = &ti;
646 	}
647 
648 	/*
649 	 * If we're re-entering the panic path, update the signature
650 	 * block so that the SC knows we're in the second part of panic.
651 	 */
652 	if (panicstr)
653 		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
654 
655 	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
656 	panic("sync initiated");
657 }
658 
659 
660 static void
startup_init(void)661 startup_init(void)
662 {
663 	/*
664 	 * We want to save the registers while we're still in OBP
665 	 * so that we know they haven't been fiddled with since.
666 	 * (In principle, OBP can't change them just because it
667 	 * makes a callback, but we'd rather not depend on that
668 	 * behavior.)
669 	 */
670 	char		sync_str[] =
671 	    "warning @ warning off : sync "
672 	    "%%tl-c %%tstate h# %p x! "
673 	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
674 	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
675 	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
676 	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
677 	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
678 	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
679 	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
680 	    "sync ; warning !";
681 
682 	/*
683 	 * 20 == num of %p substrings
684 	 * 16 == max num of chars %p will expand to.
685 	 */
686 	char 		bp[sizeof (sync_str) + 16 * 20];
687 
688 	/*
689 	 * Initialize ptl1 stack for the 1st CPU.
690 	 */
691 	ptl1_init_cpu(&cpu0);
692 
693 	/*
694 	 * Initialize the address map for cache consistent mappings
695 	 * to random pages; must be done after vac_size is set.
696 	 */
697 	ppmapinit();
698 
699 	/*
700 	 * Initialize the PROM callback handler.
701 	 */
702 	init_vx_handler();
703 
704 	/*
705 	 * have prom call sync_callback() to handle the sync and
706 	 * save some useful information which will be stored in the
707 	 * core file later.
708 	 */
709 	(void) sprintf((char *)bp, sync_str,
710 	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
711 	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
712 	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
713 	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
714 	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
715 	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
716 	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
717 	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
718 	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
719 	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
720 	prom_interpret(bp, 0, 0, 0, 0, 0);
721 	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
722 }
723 
724 
725 size_t
calc_pp_sz(pgcnt_t npages)726 calc_pp_sz(pgcnt_t npages)
727 {
728 
729 	return (npages * sizeof (struct page));
730 }
731 
732 size_t
calc_kpmpp_sz(pgcnt_t npages)733 calc_kpmpp_sz(pgcnt_t npages)
734 {
735 
736 	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
737 	kpm_pgsz = 1ull << kpm_pgshft;
738 	kpm_pgoff = kpm_pgsz - 1;
739 	kpmp2pshft = kpm_pgshft - PAGESHIFT;
740 	kpmpnpgs = 1 << kpmp2pshft;
741 
742 	if (kpm_smallpages == 0) {
743 		/*
744 		 * Avoid fragmentation problems in kphysm_init()
745 		 * by allocating for all of physical memory
746 		 */
747 		kpm_npages = ptokpmpr(physinstalled);
748 		return (kpm_npages * sizeof (kpm_page_t));
749 	} else {
750 		kpm_npages = npages;
751 		return (kpm_npages * sizeof (kpm_spage_t));
752 	}
753 }
754 
755 size_t
calc_pagehash_sz(pgcnt_t npages)756 calc_pagehash_sz(pgcnt_t npages)
757 {
758 	/* LINTED */
759 	ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
760 	/*
761 	 * The page structure hash table size is a power of 2
762 	 * such that the average hash chain length is PAGE_HASHAVELEN.
763 	 */
764 	page_hashsz = npages / PAGE_HASHAVELEN;
765 	page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
766 	    highbit(page_hashsz));
767 	page_hashsz = 1 << page_hashsz_shift;
768 	return (page_hashsz * sizeof (struct page *));
769 }
770 
771 int testkmem64_smchunks = 0;
772 
773 int
alloc_kmem64(caddr_t base,caddr_t end)774 alloc_kmem64(caddr_t base, caddr_t end)
775 {
776 	int i;
777 	caddr_t aligned_end = NULL;
778 
779 	if (testkmem64_smchunks)
780 		return (1);
781 
782 	/*
783 	 * Make one large memory alloc after figuring out the 64-bit size. This
784 	 * will enable use of the largest page size appropriate for the system
785 	 * architecture.
786 	 */
787 	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
788 	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
789 	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
790 		size_t alloc_size, alignsize;
791 #if !defined(C_OBP)
792 		unsigned long long pa;
793 #endif	/* !C_OBP */
794 
795 		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
796 			continue;
797 		alignsize = TTEBYTES(i);
798 		kmem64_szc = i;
799 
800 		/* limit page size for small memory */
801 		if (mmu_btop(alignsize) > (npages >> 2))
802 			continue;
803 
804 		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
805 		alloc_size = aligned_end - base;
806 #if !defined(C_OBP)
807 		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
808 			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
809 				kmem64_pabase = pa;
810 				kmem64_aligned_end = aligned_end;
811 				install_kmem64_tte();
812 				break;
813 			} else {
814 				prom_free_phys(alloc_size, pa);
815 			}
816 		}
817 #else	/* !C_OBP */
818 		if (prom_alloc(base, alloc_size, alignsize) == base) {
819 			kmem64_pabase = va_to_pa(kmem64_base);
820 			kmem64_aligned_end = aligned_end;
821 			break;
822 		}
823 #endif	/* !C_OBP */
824 		if (i == TTE8K) {
825 #ifdef sun4v
826 			/* return failure to try small allocations */
827 			return (1);
828 #else
829 			prom_panic("kmem64 allocation failure");
830 #endif
831 		}
832 	}
833 	ASSERT(aligned_end != NULL);
834 	return (0);
835 }
836 
837 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
838 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
839 
840 #if !defined(C_OBP)
841 /*
842  * Install a temporary tte handler in OBP for kmem64 area.
843  *
844  * We map kmem64 area with large pages before the trap table is taken
845  * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
846  * the same area. Duplicate tlb entries with different page sizes
847  * cause unpredicatble behavior.  To avoid this, we don't create
848  * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
849  * OBP).  Instead, we manage translations with a temporary va>tte-data
850  * handler (kmem64-tte).  This handler is replaced by unix-tte when
851  * the trap table is taken over.
852  *
853  * The temporary handler knows the physical address of the kmem64
854  * area. It uses the prom's pgmap@ Forth word for other addresses.
855  *
856  * We have to use BOP_ALLOC() method for C-OBP platforms because
857  * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
858  * sun4u platforms. On sun4u we flush tlb after trap table is taken
859  * over if we use large pages for kernel heap and kmem64. Since sun4u
860  * prom (unlike sun4v) calls va>tte-data first for client address
861  * translation prom's ttes for kmem64 can't get into TLB even if we
862  * later switch to prom's trap table again. C-OBP uses 4M pages for
863  * client mappings when possible so on all platforms we get the
864  * benefit from large mappings for kmem64 area immediately during
865  * boot.
866  *
867  * pseudo code:
868  * if (context != 0) {
869  * 	return false
870  * } else if (miss_va in range[kmem64_base, kmem64_end)) {
871  *	tte = tte_template +
872  *		(((miss_va & pagemask) - kmem64_base));
873  *	return tte, true
874  * } else {
875  *	return pgmap@ result
876  * }
877  */
878 char kmem64_obp_str[] =
879 	"h# %lx constant kmem64-base "
880 	"h# %lx constant kmem64-end "
881 	"h# %lx constant kmem64-pagemask "
882 	"h# %lx constant kmem64-template "
883 
884 	": kmem64-tte ( addr cnum -- false | tte-data true ) "
885 	"    if                                       ( addr ) "
886 	"       drop false exit then                  ( false ) "
887 	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
888 	"	kmem64-pagemask and                   ( addr' ) "
889 	"	kmem64-base -                         ( addr' ) "
890 	"	kmem64-template +                     ( tte ) "
891 	"	true                                  ( tte true ) "
892 	"    else                                     ( addr ) "
893 	"	pgmap@                                ( tte ) "
894 	"       dup 0< if true else drop false then   ( tte true  |  false ) "
895 	"    then                                     ( tte true  |  false ) "
896 	"; "
897 
898 	"' kmem64-tte is va>tte-data "
899 ;
900 
901 static void
install_kmem64_tte()902 install_kmem64_tte()
903 {
904 	char b[sizeof (kmem64_obp_str) + (4 * 16)];
905 	tte_t tte;
906 
907 	PRM_DEBUG(kmem64_pabase);
908 	PRM_DEBUG(kmem64_szc);
909 	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
910 	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
911 	PRM_DEBUG(tte.ll);
912 	(void) sprintf(b, kmem64_obp_str,
913 	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
914 	ASSERT(strlen(b) < sizeof (b));
915 	prom_interpret(b, 0, 0, 0, 0, 0);
916 }
917 #endif	/* !C_OBP */
918 
919 /*
920  * As OBP takes up some RAM when the system boots, pages will already be "lost"
921  * to the system and reflected in npages by the time we see it.
922  *
923  * We only want to allocate kernel structures in the 64-bit virtual address
924  * space on systems with enough RAM to make the overhead of keeping track of
925  * an extra kernel memory segment worthwhile.
926  *
927  * Since OBP has already performed its memory allocations by this point, if we
928  * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
929  * memory in the 64-bit virtual address space; otherwise keep allocations
930  * contiguous with we've mapped so far in the 32-bit virtual address space.
931  */
932 #define	MINMOVE_RAM_MB	((size_t)1900)
933 #define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
934 #define	BYTES_TO_MB(b) ((b) / 1048576ul)
935 
936 pgcnt_t	tune_npages = (pgcnt_t)
937 	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
938 
939 #pragma weak page_set_colorequiv_arr_cpu
940 extern void page_set_colorequiv_arr_cpu(void);
941 extern void page_set_colorequiv_arr(void);
942 
943 static pgcnt_t ramdisk_npages;
944 static struct memlist *old_phys_avail;
945 
946 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
947 
948 static void
startup_memlist(void)949 startup_memlist(void)
950 {
951 	size_t hmehash_sz, pagelist_sz, tt_sz;
952 	size_t psetable_sz;
953 	caddr_t alloc_base;
954 	caddr_t memspace;
955 	struct memlist *cur;
956 	size_t syslimit = (size_t)SYSLIMIT;
957 	size_t sysbase = (size_t)SYSBASE;
958 
959 	/*
960 	 * Initialize enough of the system to allow kmem_alloc to work by
961 	 * calling boot to allocate its memory until the time that
962 	 * kvm_init is completed.  The page structs are allocated after
963 	 * rounding up end to the nearest page boundary; the memsegs are
964 	 * initialized and the space they use comes from the kernel heap.
965 	 * With appropriate initialization, they can be reallocated later
966 	 * to a size appropriate for the machine's configuration.
967 	 *
968 	 * At this point, memory is allocated for things that will never
969 	 * need to be freed, this used to be "valloced".  This allows a
970 	 * savings as the pages don't need page structures to describe
971 	 * them because them will not be managed by the vm system.
972 	 */
973 
974 	/*
975 	 * We're loaded by boot with the following configuration (as
976 	 * specified in the sun4u/conf/Mapfile):
977 	 *
978 	 * 	text:		4 MB chunk aligned on a 4MB boundary
979 	 * 	data & bss:	4 MB chunk aligned on a 4MB boundary
980 	 *
981 	 * These two chunks will eventually be mapped by 2 locked 4MB
982 	 * ttes and will represent the nucleus of the kernel.  This gives
983 	 * us some free space that is already allocated, some or all of
984 	 * which is made available to kernel module text.
985 	 *
986 	 * The free space in the data-bss chunk is used for nucleus
987 	 * allocatable data structures and we reserve it using the
988 	 * nalloc_base and nalloc_end variables.  This space is currently
989 	 * being used for hat data structures required for tlb miss
990 	 * handling operations.  We align nalloc_base to a l2 cache
991 	 * linesize because this is the line size the hardware uses to
992 	 * maintain cache coherency.
993 	 * 512K is carved out for module data.
994 	 */
995 
996 	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
997 	e_moddata = moddata + MODDATA;
998 	nalloc_base = e_moddata;
999 
1000 	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
1001 	valloc_base = nalloc_base;
1002 
1003 	/*
1004 	 * Calculate the start of the data segment.
1005 	 */
1006 	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1007 		prom_panic("nucleus data overflow");
1008 
1009 	PRM_DEBUG(moddata);
1010 	PRM_DEBUG(nalloc_base);
1011 	PRM_DEBUG(nalloc_end);
1012 
1013 	/*
1014 	 * Remember any slop after e_text so we can give it to the modules.
1015 	 */
1016 	PRM_DEBUG(e_text);
1017 	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1018 	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1019 		prom_panic("nucleus text overflow");
1020 	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1021 	    modtext;
1022 	PRM_DEBUG(modtext);
1023 	PRM_DEBUG(modtext_sz);
1024 
1025 	init_boot_memlists();
1026 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1027 	    &boot_physavail, &boot_physavail_len,
1028 	    &boot_virtavail, &boot_virtavail_len);
1029 
1030 	/*
1031 	 * Remember what the physically available highest page is
1032 	 * so that dumpsys works properly, and find out how much
1033 	 * memory is installed.
1034 	 */
1035 	installed_top_size_memlist_array(boot_physinstalled,
1036 	    boot_physinstalled_len, &physmax, &physinstalled);
1037 	PRM_DEBUG(physinstalled);
1038 	PRM_DEBUG(physmax);
1039 
1040 	/* Fill out memory nodes config structure */
1041 	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1042 
1043 	/*
1044 	 * npages is the maximum of available physical memory possible.
1045 	 * (ie. it will never be more than this)
1046 	 *
1047 	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1048 	 * using phys_avail will underestimate what will end up being freed.
1049 	 * A better initial guess is just total memory minus the kernel text
1050 	 */
1051 	npages = physinstalled - btop(MMU_PAGESIZE4M);
1052 
1053 	/*
1054 	 * First allocate things that can go in the nucleus data page
1055 	 * (fault status, TSBs, dmv, CPUs)
1056 	 */
1057 	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1058 
1059 	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1060 		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1061 
1062 	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1063 		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1064 
1065 	if (ndata_alloc_dmv(&ndata) != 0)
1066 		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1067 
1068 	if (ndata_alloc_page_mutexs(&ndata) != 0)
1069 		cmn_err(CE_PANIC,
1070 		    "no more nucleus memory after page free lists alloc");
1071 
1072 	if (ndata_alloc_hat(&ndata) != 0)
1073 		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1074 
1075 	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1076 		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1077 
1078 	/*
1079 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1080 	 *
1081 	 * There are comments all over the SFMMU code warning of dire
1082 	 * consequences if the TSBs are moved out of 32-bit space.  This
1083 	 * is largely because the asm code uses "sethi %hi(addr)"-type
1084 	 * instructions which will not provide the expected result if the
1085 	 * address is a 64-bit one.
1086 	 *
1087 	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1088 	 */
1089 	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1090 	PRM_DEBUG(alloc_base);
1091 
1092 	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1093 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1094 	PRM_DEBUG(alloc_base);
1095 
1096 	/*
1097 	 * Allocate IOMMU TSB array.  We do this here so that the physical
1098 	 * memory gets deducted from the PROM's physical memory list.
1099 	 */
1100 	alloc_base = iommu_tsb_init(alloc_base);
1101 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1102 	PRM_DEBUG(alloc_base);
1103 
1104 	/*
1105 	 * Allow for an early allocation of physically contiguous memory.
1106 	 */
1107 	alloc_base = contig_mem_prealloc(alloc_base, npages);
1108 
1109 	/*
1110 	 * Platforms like Starcat and OPL need special structures assigned in
1111 	 * 32-bit virtual address space because their probing routines execute
1112 	 * FCode, and FCode can't handle 64-bit virtual addresses...
1113 	 */
1114 	if (&plat_startup_memlist) {
1115 		alloc_base = plat_startup_memlist(alloc_base);
1116 		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1117 		    ecache_alignsize);
1118 		PRM_DEBUG(alloc_base);
1119 	}
1120 
1121 	/*
1122 	 * Save off where the contiguous allocations to date have ended
1123 	 * in econtig32.
1124 	 */
1125 	econtig32 = alloc_base;
1126 	PRM_DEBUG(econtig32);
1127 	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1128 		cmn_err(CE_PANIC, "econtig32 too big");
1129 
1130 	pp_sz = calc_pp_sz(npages);
1131 	PRM_DEBUG(pp_sz);
1132 	if (kpm_enable) {
1133 		kpm_pp_sz = calc_kpmpp_sz(npages);
1134 		PRM_DEBUG(kpm_pp_sz);
1135 	}
1136 
1137 	hmehash_sz = calc_hmehash_sz(npages);
1138 	PRM_DEBUG(hmehash_sz);
1139 
1140 	pagehash_sz = calc_pagehash_sz(npages);
1141 	PRM_DEBUG(pagehash_sz);
1142 
1143 	pagelist_sz = calc_free_pagelist_sz();
1144 	PRM_DEBUG(pagelist_sz);
1145 
1146 #ifdef	TRAPTRACE
1147 	tt_sz = calc_traptrace_sz();
1148 	PRM_DEBUG(tt_sz);
1149 #else
1150 	tt_sz = 0;
1151 #endif	/* TRAPTRACE */
1152 
1153 	/*
1154 	 * Place the array that protects pp->p_selock in the kmem64 wad.
1155 	 */
1156 	pse_shift = size_pse_array(npages, max_ncpus);
1157 	PRM_DEBUG(pse_shift);
1158 	pse_table_size = 1 << pse_shift;
1159 	PRM_DEBUG(pse_table_size);
1160 	psetable_sz = roundup(
1161 	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1162 	PRM_DEBUG(psetable_sz);
1163 
1164 	/*
1165 	 * Now allocate the whole wad
1166 	 */
1167 	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1168 	    pagelist_sz + tt_sz + psetable_sz;
1169 	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1170 	kmem64_base = (caddr_t)syslimit;
1171 	kmem64_end = kmem64_base + kmem64_sz;
1172 	if (alloc_kmem64(kmem64_base, kmem64_end)) {
1173 		/*
1174 		 * Attempt for kmem64 to allocate one big
1175 		 * contiguous chunk of memory failed.
1176 		 * We get here because we are sun4v.
1177 		 * We will proceed by breaking up
1178 		 * the allocation into two attempts.
1179 		 * First, we allocate kpm_pp_sz, hmehash_sz,
1180 		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1181 		 * one contiguous chunk. This is a much smaller
1182 		 * chunk and we should get it, if not we panic.
1183 		 * Note that hmehash and tt need to be physically
1184 		 * (in the real address sense) contiguous.
1185 		 * Next, we use bop_alloc_chunk() to
1186 		 * to allocate the page_t structures.
1187 		 * This will allow the page_t to be allocated
1188 		 * in multiple smaller chunks.
1189 		 * In doing so, the assumption that page_t is
1190 		 * physically contiguous no longer hold, this is ok
1191 		 * for sun4v but not for sun4u.
1192 		 */
1193 		size_t  tmp_size;
1194 		caddr_t tmp_base;
1195 
1196 		pp_sz  = roundup(pp_sz, PAGESIZE);
1197 
1198 		/*
1199 		 * Allocate kpm_pp_sz, hmehash_sz,
1200 		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1201 		 */
1202 		tmp_base = kmem64_base + pp_sz;
1203 		tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1204 		    pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1205 		if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1206 			prom_panic("kmem64 prom_alloc contig failed");
1207 		PRM_DEBUG(tmp_base);
1208 		PRM_DEBUG(tmp_size);
1209 
1210 		/*
1211 		 * Allocate the page_ts
1212 		 */
1213 		if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1214 			prom_panic("kmem64 bop_alloc_chunk page_t failed");
1215 		PRM_DEBUG(kmem64_base);
1216 		PRM_DEBUG(pp_sz);
1217 
1218 		kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1219 		ASSERT(kmem64_aligned_end >= kmem64_end);
1220 
1221 		kmem64_smchunks = 1;
1222 	} else {
1223 
1224 		/*
1225 		 * We need to adjust pp_sz for the normal
1226 		 * case where kmem64 can allocate one large chunk
1227 		 */
1228 		if (kpm_smallpages == 0) {
1229 			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1230 		} else {
1231 			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1232 			    sizeof (kpm_spage_t));
1233 		}
1234 		pp_sz = npages * sizeof (struct page);
1235 	}
1236 
1237 	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1238 		cmn_err(CE_PANIC, "not enough kmem64 space");
1239 	PRM_DEBUG(kmem64_base);
1240 	PRM_DEBUG(kmem64_end);
1241 	PRM_DEBUG(kmem64_aligned_end);
1242 
1243 	/*
1244 	 * ... and divy it up
1245 	 */
1246 	alloc_base = kmem64_base;
1247 
1248 	pp_base = (page_t *)alloc_base;
1249 	alloc_base += pp_sz;
1250 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1251 	PRM_DEBUG(pp_base);
1252 	PRM_DEBUG(npages);
1253 
1254 	if (kpm_enable) {
1255 		kpm_pp_base = alloc_base;
1256 		if (kpm_smallpages == 0) {
1257 			/* kpm_npages based on physinstalled, don't reset */
1258 			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1259 		} else {
1260 			kpm_npages = ptokpmpr(npages);
1261 			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1262 		}
1263 		alloc_base += kpm_pp_sz;
1264 		alloc_base =
1265 		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1266 		PRM_DEBUG(kpm_pp_base);
1267 	}
1268 
1269 	alloc_base = alloc_hmehash(alloc_base);
1270 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1271 	PRM_DEBUG(alloc_base);
1272 
1273 	page_hash = (page_t **)alloc_base;
1274 	alloc_base += pagehash_sz;
1275 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1276 	PRM_DEBUG(page_hash);
1277 
1278 	alloc_base = alloc_page_freelists(alloc_base);
1279 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1280 	PRM_DEBUG(alloc_base);
1281 
1282 #ifdef	TRAPTRACE
1283 	ttrace_buf = alloc_base;
1284 	alloc_base += tt_sz;
1285 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1286 	PRM_DEBUG(alloc_base);
1287 #endif	/* TRAPTRACE */
1288 
1289 	pse_mutex = (pad_mutex_t *)alloc_base;
1290 	alloc_base += psetable_sz;
1291 	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1292 	PRM_DEBUG(alloc_base);
1293 
1294 	/*
1295 	 * Note that if we use small chunk allocations for
1296 	 * kmem64, we need to ensure kmem64_end is the same as
1297 	 * kmem64_aligned_end to prevent subsequent logic from
1298 	 * trying to reuse the overmapping.
1299 	 * Otherwise we adjust kmem64_end to what we really allocated.
1300 	 */
1301 	if (kmem64_smchunks) {
1302 		kmem64_end = kmem64_aligned_end;
1303 	} else {
1304 		kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1305 	}
1306 	kmem64_sz = kmem64_end - kmem64_base;
1307 
1308 	if (&ecache_init_scrub_flush_area) {
1309 		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1310 		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1311 	}
1312 
1313 	/*
1314 	 * If physmem is patched to be non-zero, use it instead of
1315 	 * the monitor value unless physmem is larger than the total
1316 	 * amount of memory on hand.
1317 	 */
1318 	if (physmem == 0 || physmem > npages)
1319 		physmem = npages;
1320 
1321 	/*
1322 	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1323 	 * is mounted as root. This memory is held down by OBP and unlike
1324 	 * the stub boot_archive is never released.
1325 	 *
1326 	 * In order to get things sized correctly on lower memory
1327 	 * machines (where the memory used by the ramdisk represents
1328 	 * a significant portion of memory), physmem is adjusted.
1329 	 *
1330 	 * This is done by subtracting the ramdisk_size which is set
1331 	 * to the size of the ramdisk (in Kb) in /etc/system at the
1332 	 * time the miniroot archive is constructed.
1333 	 */
1334 	if (root_is_ramdisk == B_TRUE) {
1335 		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1336 		physmem -= ramdisk_npages;
1337 	}
1338 
1339 	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1340 		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1341 
1342 	/*
1343 	 * Allocate space for the interrupt vector table.
1344 	 */
1345 	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1346 	if (memspace != (caddr_t)intr_vec_table)
1347 		prom_panic("interrupt vector table allocation failure");
1348 
1349 	/*
1350 	 * Between now and when we finish copying in the memory lists,
1351 	 * allocations happen so the space gets fragmented and the
1352 	 * lists longer.  Leave enough space for lists twice as
1353 	 * long as we have now; then roundup to a pagesize.
1354 	 */
1355 	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1356 	    prom_phys_avail_len() + prom_virt_avail_len());
1357 	memlist_sz *= 2;
1358 	memlist_sz = roundup(memlist_sz, PAGESIZE);
1359 	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1360 	if (memspace == NULL)
1361 		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1362 
1363 	memlist = (struct memlist *)memspace;
1364 	memlist_end = (char *)memspace + memlist_sz;
1365 	PRM_DEBUG(memlist);
1366 	PRM_DEBUG(memlist_end);
1367 
1368 	PRM_DEBUG(sysbase);
1369 	PRM_DEBUG(syslimit);
1370 	kernelheap_init((void *)sysbase, (void *)syslimit,
1371 	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1372 
1373 	/*
1374 	 * Take the most current snapshot we can by calling mem-update.
1375 	 */
1376 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1377 	    &boot_physavail, &boot_physavail_len,
1378 	    &boot_virtavail, &boot_virtavail_len);
1379 
1380 	/*
1381 	 * Remove the space used by prom_alloc from the kernel heap
1382 	 * plus the area actually used by the OBP (if any)
1383 	 * ignoring virtual addresses in virt_avail, above syslimit.
1384 	 */
1385 	virt_avail = memlist;
1386 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1387 
1388 	for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1389 		uint64_t range_base, range_size;
1390 
1391 		if ((range_base = cur->ml_address + cur->ml_size) <
1392 		    (uint64_t)sysbase)
1393 			continue;
1394 		if (range_base >= (uint64_t)syslimit)
1395 			break;
1396 		/*
1397 		 * Limit the range to end at syslimit.
1398 		 */
1399 		range_size = MIN(cur->ml_next->ml_address,
1400 		    (uint64_t)syslimit) - range_base;
1401 		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1402 		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1403 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1404 	}
1405 
1406 	phys_avail = memlist;
1407 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1408 
1409 	/*
1410 	 * Add any extra memory at the end of the ndata region if there's at
1411 	 * least a page to add.  There might be a few more pages available in
1412 	 * the middle of the ndata region, but for now they are ignored.
1413 	 */
1414 	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1415 	if (nalloc_base == NULL)
1416 		nalloc_base = nalloc_end;
1417 	ndata_remain_sz = nalloc_end - nalloc_base;
1418 
1419 	/*
1420 	 * Copy physinstalled list into kernel space.
1421 	 */
1422 	phys_install = memlist;
1423 	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1424 
1425 	/*
1426 	 * Create list of physical addrs we don't need pp's for:
1427 	 * kernel text 4M page
1428 	 * kernel data 4M page - ndata_remain_sz
1429 	 * kmem64 pages
1430 	 *
1431 	 * NB if adding any pages here, make sure no kpm page
1432 	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1433 	 */
1434 	nopp_list = memlist;
1435 	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1436 	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1437 	    &memlist, &nopp_list);
1438 
1439 	/* Don't add to nopp_list if kmem64 was allocated in smchunks */
1440 	if (!kmem64_smchunks)
1441 		memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1442 
1443 	if ((caddr_t)memlist > (memspace + memlist_sz))
1444 		prom_panic("memlist overflow");
1445 
1446 	/*
1447 	 * Size the pcf array based on the number of cpus in the box at
1448 	 * boot time.
1449 	 */
1450 	pcf_init();
1451 
1452 	/*
1453 	 * Initialize the page structures from the memory lists.
1454 	 */
1455 	kphysm_init();
1456 
1457 	availrmem_initial = availrmem = freemem;
1458 	PRM_DEBUG(availrmem);
1459 
1460 	/*
1461 	 * Some of the locks depend on page_hashsz being set!
1462 	 * kmem_init() depends on this; so, keep it here.
1463 	 */
1464 	page_lock_init();
1465 
1466 	/*
1467 	 * Initialize kernel memory allocator.
1468 	 */
1469 	kmem_init();
1470 
1471 	/*
1472 	 * Factor in colorequiv to check additional 'equivalent' bins
1473 	 */
1474 	if (&page_set_colorequiv_arr_cpu != NULL)
1475 		page_set_colorequiv_arr_cpu();
1476 	else
1477 		page_set_colorequiv_arr();
1478 
1479 	/*
1480 	 * Initialize bp_mapin().
1481 	 */
1482 	bp_init(shm_alignment, HAT_STRICTORDER);
1483 
1484 	/*
1485 	 * Reserve space for MPO mblock structs from the 32-bit heap.
1486 	 */
1487 
1488 	if (mpo_heap32_bufsz > (size_t)0) {
1489 		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1490 		    PAGESIZE, 0, 0, mpo_heap32_buf,
1491 		    mpo_heap32_buf + mpo_heap32_bufsz,
1492 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1493 	}
1494 	mem_config_init();
1495 }
1496 
1497 static void
startup_modules(void)1498 startup_modules(void)
1499 {
1500 	int nhblk1, nhblk8;
1501 	size_t  nhblksz;
1502 	pgcnt_t pages_per_hblk;
1503 	size_t hme8blk_sz, hme1blk_sz;
1504 
1505 	/*
1506 	 * The system file /etc/system was read already under startup_memlist.
1507 	 */
1508 	if (&set_platform_defaults)
1509 		set_platform_defaults();
1510 
1511 	/*
1512 	 * Calculate default settings of system parameters based upon
1513 	 * maxusers, yet allow to be overridden via the /etc/system file.
1514 	 */
1515 	param_calc(0);
1516 
1517 	mod_setup();
1518 
1519 	/*
1520 	 * If this is a positron, complain and halt.
1521 	 */
1522 	if (&iam_positron && iam_positron()) {
1523 		cmn_err(CE_WARN, "This hardware platform is not supported"
1524 		    " by this release of Solaris.\n");
1525 #ifdef DEBUG
1526 		prom_enter_mon();	/* Type 'go' to resume */
1527 		cmn_err(CE_WARN, "Booting an unsupported platform.\n");
1528 		cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
1529 
1530 #else /* DEBUG */
1531 		halt(0);
1532 #endif /* DEBUG */
1533 	}
1534 
1535 	/*
1536 	 * If we are running firmware that isn't 64-bit ready
1537 	 * then complain and halt.
1538 	 */
1539 	do_prom_version_check();
1540 
1541 	/*
1542 	 * Initialize system parameters
1543 	 */
1544 	param_init();
1545 
1546 	/*
1547 	 * maxmem is the amount of physical memory we're playing with.
1548 	 */
1549 	maxmem = physmem;
1550 
1551 	/* Set segkp limits. */
1552 	ncbase = kdi_segdebugbase;
1553 	ncend = kdi_segdebugbase;
1554 
1555 	/*
1556 	 * Initialize the hat layer.
1557 	 */
1558 	hat_init();
1559 
1560 	/*
1561 	 * Initialize segment management stuff.
1562 	 */
1563 	seg_init();
1564 
1565 	/*
1566 	 * Create the va>tte handler, so the prom can understand
1567 	 * kernel translations.  The handler is installed later, just
1568 	 * as we are about to take over the trap table from the prom.
1569 	 */
1570 	create_va_to_tte();
1571 
1572 	/*
1573 	 * Load the forthdebugger (optional)
1574 	 */
1575 	forthdebug_init();
1576 
1577 	/*
1578 	 * Create OBP node for console input callbacks
1579 	 * if it is needed.
1580 	 */
1581 	startup_create_io_node();
1582 
1583 	if (modloadonly("fs", "specfs") == -1)
1584 		halt("Can't load specfs");
1585 
1586 	if (modloadonly("fs", "devfs") == -1)
1587 		halt("Can't load devfs");
1588 
1589 	if (modloadonly("fs", "procfs") == -1)
1590 		halt("Can't load procfs");
1591 
1592 	if (modloadonly("misc", "swapgeneric") == -1)
1593 		halt("Can't load swapgeneric");
1594 
1595 	(void) modloadonly("sys", "lbl_edition");
1596 
1597 	dispinit();
1598 
1599 	/*
1600 	 * Infer meanings to the members of the idprom buffer.
1601 	 */
1602 	parse_idprom();
1603 
1604 	/* Read cluster configuration data. */
1605 	clconf_init();
1606 
1607 	setup_ddi();
1608 
1609 	/*
1610 	 * Lets take this opportunity to load the root device.
1611 	 */
1612 	if (loadrootmodules() != 0)
1613 		debug_enter("Can't load the root filesystem");
1614 
1615 	/*
1616 	 * Load tod driver module for the tod part found on this system.
1617 	 * Recompute the cpu frequency/delays based on tod as tod part
1618 	 * tends to keep time more accurately.
1619 	 */
1620 	if (&load_tod_module)
1621 		load_tod_module();
1622 
1623 	/*
1624 	 * Allow platforms to load modules which might
1625 	 * be needed after bootops are gone.
1626 	 */
1627 	if (&load_platform_modules)
1628 		load_platform_modules();
1629 
1630 	setcpudelay();
1631 
1632 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1633 	    &boot_physavail, &boot_physavail_len,
1634 	    &boot_virtavail, &boot_virtavail_len);
1635 
1636 	/*
1637 	 * Calculation and allocation of hmeblks needed to remap
1638 	 * the memory allocated by PROM till now.
1639 	 * Overestimate the number of hblk1 elements by assuming
1640 	 * worst case of TTE64K mappings.
1641 	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1642 	 */
1643 	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1644 	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1645 	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1646 	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1647 
1648 	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1649 
1650 	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1651 	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1652 	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1653 
1654 	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1655 
1656 	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1657 	nhblk8 = 0;
1658 	while (bop_alloc_pages > 1) {
1659 		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1660 		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1661 		bop_alloc_pages *= hme8blk_sz;
1662 		bop_alloc_pages = btopr(bop_alloc_pages);
1663 	}
1664 	nhblk8 += 2;
1665 
1666 	/*
1667 	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1668 	 * boundary, the number of hblk8's needed to map the entries in the
1669 	 * boot_virtavail list needs to be adjusted to take this into
1670 	 * consideration.  Thus, we need to add additional hblk8's since it
1671 	 * is possible that an hblk8 will not have all 8 slots used due to
1672 	 * alignment constraints.  Since there were boot_virtavail_len entries
1673 	 * in that list, we need to add that many hblk8's to the number
1674 	 * already calculated to make sure we don't underestimate.
1675 	 */
1676 	nhblk8 += boot_virtavail_len;
1677 	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1678 
1679 	/* Allocate in pagesize chunks */
1680 	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1681 	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1682 	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1683 }
1684 
1685 static void
startup_bop_gone(void)1686 startup_bop_gone(void)
1687 {
1688 
1689 	/*
1690 	 * Destroy the MD initialized at startup
1691 	 * The startup initializes the MD framework
1692 	 * using prom and BOP alloc free it now.
1693 	 */
1694 	mach_descrip_startup_fini();
1695 
1696 	/*
1697 	 * We're done with prom allocations.
1698 	 */
1699 	bop_fini();
1700 
1701 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1702 	    &boot_physavail, &boot_physavail_len,
1703 	    &boot_virtavail, &boot_virtavail_len);
1704 
1705 	/*
1706 	 * setup physically contiguous area twice as large as the ecache.
1707 	 * this is used while doing displacement flush of ecaches
1708 	 */
1709 	if (&ecache_flush_address) {
1710 		ecache_flushaddr = ecache_flush_address();
1711 		if (ecache_flushaddr == (uint64_t)-1) {
1712 			cmn_err(CE_PANIC,
1713 			    "startup: no memory to set ecache_flushaddr");
1714 		}
1715 	}
1716 
1717 	/*
1718 	 * Virtual available next.
1719 	 */
1720 	ASSERT(virt_avail != NULL);
1721 	memlist_free_list(virt_avail);
1722 	virt_avail = memlist;
1723 	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1724 
1725 }
1726 
1727 
1728 /*
1729  * startup_fixup_physavail - called from mach_sfmmu.c after the final
1730  * allocations have been performed.  We can't call it in startup_bop_gone
1731  * since later operations can cause obp to allocate more memory.
1732  */
1733 void
startup_fixup_physavail(void)1734 startup_fixup_physavail(void)
1735 {
1736 	struct memlist *cur;
1737 	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1738 
1739 	PRM_DEBUG(kmem64_overmap_size);
1740 
1741 	/*
1742 	 * take the most current snapshot we can by calling mem-update
1743 	 */
1744 	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1745 	    &boot_physavail, &boot_physavail_len,
1746 	    &boot_virtavail, &boot_virtavail_len);
1747 
1748 	/*
1749 	 * Copy phys_avail list, again.
1750 	 * Both the kernel/boot and the prom have been allocating
1751 	 * from the original list we copied earlier.
1752 	 */
1753 	cur = memlist;
1754 	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1755 
1756 	/*
1757 	 * Add any unused kmem64 memory from overmapped page
1758 	 * (Note: va_to_pa does not work for kmem64_end)
1759 	 */
1760 	if (kmem64_overmap_size) {
1761 		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1762 		    kmem64_overmap_size, &memlist, &cur);
1763 	}
1764 
1765 	/*
1766 	 * Add any extra memory after e_data we added to the phys_avail list
1767 	 * back to the old list.
1768 	 */
1769 	if (ndata_remain_sz >= MMU_PAGESIZE)
1770 		memlist_add(va_to_pa(nalloc_base),
1771 		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1772 
1773 	/*
1774 	 * There isn't any bounds checking on the memlist area
1775 	 * so ensure it hasn't overgrown.
1776 	 */
1777 	if ((caddr_t)memlist > (caddr_t)memlist_end)
1778 		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1779 
1780 	/*
1781 	 * The kernel removes the pages that were allocated for it from
1782 	 * the freelist, but we now have to find any -extra- pages that
1783 	 * the prom has allocated for it's own book-keeping, and remove
1784 	 * them from the freelist too. sigh.
1785 	 */
1786 	sync_memlists(phys_avail, cur);
1787 
1788 	ASSERT(phys_avail != NULL);
1789 
1790 	old_phys_avail = phys_avail;
1791 	phys_avail = cur;
1792 }
1793 
1794 void
update_kcage_ranges(uint64_t addr,uint64_t len)1795 update_kcage_ranges(uint64_t addr, uint64_t len)
1796 {
1797 	pfn_t base = btop(addr);
1798 	pgcnt_t num = btop(len);
1799 	int rv;
1800 
1801 	rv = kcage_range_add(base, num, kcage_startup_dir);
1802 
1803 	if (rv == ENOMEM) {
1804 		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1805 		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1806 	} else if (rv != 0) {
1807 		/* catch this in debug kernels */
1808 		ASSERT(0);
1809 
1810 		cmn_err(CE_WARN, "unexpected kcage_range_add"
1811 		    " return value %d", rv);
1812 	}
1813 }
1814 
1815 static void
startup_vm(void)1816 startup_vm(void)
1817 {
1818 	size_t	i;
1819 	struct segmap_crargs a;
1820 	struct segkpm_crargs b;
1821 
1822 	uint64_t avmem;
1823 	caddr_t va;
1824 	pgcnt_t	max_phys_segkp;
1825 	int	mnode;
1826 
1827 	extern int use_brk_lpg, use_stk_lpg;
1828 
1829 	/*
1830 	 * get prom's mappings, create hments for them and switch
1831 	 * to the kernel context.
1832 	 */
1833 	hat_kern_setup();
1834 
1835 	/*
1836 	 * Take over trap table
1837 	 */
1838 	setup_trap_table();
1839 
1840 	/*
1841 	 * Install the va>tte handler, so that the prom can handle
1842 	 * misses and understand the kernel table layout in case
1843 	 * we need call into the prom.
1844 	 */
1845 	install_va_to_tte();
1846 
1847 	/*
1848 	 * Set a flag to indicate that the tba has been taken over.
1849 	 */
1850 	tba_taken_over = 1;
1851 
1852 	/* initialize MMU primary context register */
1853 	mmu_init_kcontext();
1854 
1855 	/*
1856 	 * The boot cpu can now take interrupts, x-calls, x-traps
1857 	 */
1858 	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1859 	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1860 
1861 	/*
1862 	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1863 	 */
1864 	tbr_wr_addr_inited = 1;
1865 
1866 	/*
1867 	 * Initialize VM system, and map kernel address space.
1868 	 */
1869 	kvm_init();
1870 
1871 	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1872 	if (kernel_cage_enable) {
1873 		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1874 	}
1875 	memlist_free_list(old_phys_avail);
1876 
1877 	/*
1878 	 * If the following is true, someone has patched
1879 	 * phsymem to be less than the number of pages that
1880 	 * the system actually has.  Remove pages until system
1881 	 * memory is limited to the requested amount.  Since we
1882 	 * have allocated page structures for all pages, we
1883 	 * correct the amount of memory we want to remove
1884 	 * by the size of the memory used to hold page structures
1885 	 * for the non-used pages.
1886 	 */
1887 	if (physmem + ramdisk_npages < npages) {
1888 		pgcnt_t diff, off;
1889 		struct page *pp;
1890 		struct seg kseg;
1891 
1892 		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1893 
1894 		off = 0;
1895 		diff = npages - (physmem + ramdisk_npages);
1896 		diff -= mmu_btopr(diff * sizeof (struct page));
1897 		kseg.s_as = &kas;
1898 		while (diff--) {
1899 			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1900 			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1901 			    &kseg, (caddr_t)off);
1902 			if (pp == NULL)
1903 				cmn_err(CE_PANIC, "limited physmem too much!");
1904 			page_io_unlock(pp);
1905 			page_downgrade(pp);
1906 			availrmem--;
1907 			off += MMU_PAGESIZE;
1908 		}
1909 	}
1910 
1911 	/*
1912 	 * When printing memory, show the total as physmem less
1913 	 * that stolen by a debugger.
1914 	 */
1915 	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1916 	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1917 	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1918 
1919 	avmem = (uint64_t)freemem << PAGESHIFT;
1920 	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1921 
1922 	/*
1923 	 * For small memory systems disable automatic large pages.
1924 	 */
1925 	if (physmem < privm_lpg_min_physmem) {
1926 		use_brk_lpg = 0;
1927 		use_stk_lpg = 0;
1928 	}
1929 
1930 	/*
1931 	 * Perform platform specific freelist processing
1932 	 */
1933 	if (&plat_freelist_process) {
1934 		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1935 			if (mem_node_config[mnode].exists)
1936 				plat_freelist_process(mnode);
1937 	}
1938 
1939 	/*
1940 	 * Initialize the segkp segment type.  We position it
1941 	 * after the configured tables and buffers (whose end
1942 	 * is given by econtig) and before V_WKBASE_ADDR.
1943 	 * Also in this area is segkmap (size SEGMAPSIZE).
1944 	 */
1945 
1946 	/* XXX - cache alignment? */
1947 	va = (caddr_t)SEGKPBASE;
1948 	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1949 
1950 	max_phys_segkp = (physmem * 2);
1951 
1952 	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1953 		segkpsize = btop(SEGKPDEFSIZE);
1954 		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1955 		    "segkpsize has been reset to %ld pages", segkpsize);
1956 	}
1957 
1958 	i = ptob(MIN(segkpsize, max_phys_segkp));
1959 
1960 	rw_enter(&kas.a_lock, RW_WRITER);
1961 	if (seg_attach(&kas, va, i, segkp) < 0)
1962 		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1963 	if (segkp_create(segkp) != 0)
1964 		cmn_err(CE_PANIC, "startup: segkp_create failed");
1965 	rw_exit(&kas.a_lock);
1966 
1967 	/*
1968 	 * kpm segment
1969 	 */
1970 	segmap_kpm = kpm_enable &&
1971 	    segmap_kpm && PAGESIZE == MAXBSIZE;
1972 
1973 	if (kpm_enable) {
1974 		rw_enter(&kas.a_lock, RW_WRITER);
1975 
1976 		/*
1977 		 * The segkpm virtual range range is larger than the
1978 		 * actual physical memory size and also covers gaps in
1979 		 * the physical address range for the following reasons:
1980 		 * . keep conversion between segkpm and physical addresses
1981 		 *   simple, cheap and unambiguous.
1982 		 * . avoid extension/shrink of the the segkpm in case of DR.
1983 		 * . avoid complexity for handling of virtual addressed
1984 		 *   caches, segkpm and the regular mapping scheme must be
1985 		 *   kept in sync wrt. the virtual color of mapped pages.
1986 		 * Any accesses to virtual segkpm ranges not backed by
1987 		 * physical memory will fall through the memseg pfn hash
1988 		 * and will be handled in segkpm_fault.
1989 		 * Additional kpm_size spaces needed for vac alias prevention.
1990 		 */
1991 		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1992 		    segkpm) < 0)
1993 			cmn_err(CE_PANIC, "cannot attach segkpm");
1994 
1995 		b.prot = PROT_READ | PROT_WRITE;
1996 		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1997 
1998 		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1999 			panic("segkpm_create segkpm");
2000 
2001 		rw_exit(&kas.a_lock);
2002 
2003 		mach_kpm_init();
2004 	}
2005 
2006 	va = kpm_vbase + (kpm_size * vac_colors);
2007 
2008 	if (!segzio_fromheap) {
2009 		size_t size;
2010 		size_t physmem_b = mmu_ptob(physmem);
2011 
2012 		/* size is in bytes, segziosize is in pages */
2013 		if (segziosize == 0) {
2014 			size = physmem_b;
2015 		} else {
2016 			size = mmu_ptob(segziosize);
2017 		}
2018 
2019 		if (size < SEGZIOMINSIZE) {
2020 			size = SEGZIOMINSIZE;
2021 		} else if (size > SEGZIOMAXSIZE) {
2022 			size = SEGZIOMAXSIZE;
2023 			/*
2024 			 * On 64-bit x86, we only have 2TB of KVA.  This exists
2025 			 * for parity with x86.
2026 			 *
2027 			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2028 			 * doesn't consume all of KVA.  However, if we have a
2029 			 * system that has more thant 512gb of physical memory,
2030 			 * we can actually consume about half of the difference
2031 			 * between 512gb and the rest of the available physical
2032 			 * memory.
2033 			 */
2034 			if (physmem_b > SEGZIOMAXSIZE) {
2035 				size += (physmem_b - SEGZIOMAXSIZE) / 2;
2036 		}
2037 		}
2038 		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2039 		/* put the base of the ZIO segment after the kpm segment */
2040 		segzio_base = va;
2041 		va += mmu_ptob(segziosize);
2042 		PRM_DEBUG(segziosize);
2043 		PRM_DEBUG(segzio_base);
2044 
2045 		/*
2046 		 * On some platforms, kvm_init is called after the kpm
2047 		 * sizes have been determined.  On SPARC, kvm_init is called
2048 		 * before, so we have to attach the kzioseg after kvm is
2049 		 * initialized, otherwise we'll try to allocate from the boot
2050 		 * area since the kernel heap hasn't yet been configured.
2051 		 */
2052 		rw_enter(&kas.a_lock, RW_WRITER);
2053 
2054 		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2055 		    &kzioseg);
2056 		(void) segkmem_zio_create(&kzioseg);
2057 
2058 		/* create zio area covering new segment */
2059 		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2060 
2061 		rw_exit(&kas.a_lock);
2062 	}
2063 
2064 	if (ppvm_enable) {
2065 		caddr_t ppvm_max;
2066 
2067 		/*
2068 		 * ppvm refers to the static VA space used to map
2069 		 * the page_t's for dynamically added memory.
2070 		 *
2071 		 * ppvm_base should not cross a potential VA hole.
2072 		 *
2073 		 * ppvm_size should be large enough to map the
2074 		 * page_t's needed to manage all of KPM range.
2075 		 */
2076 		ppvm_size =
2077 		    roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2078 		    MMU_PAGESIZE);
2079 		ppvm_max = (caddr_t)(0ull - ppvm_size);
2080 		ppvm_base = (page_t *)va;
2081 
2082 		if ((caddr_t)ppvm_base <= hole_end) {
2083 			cmn_err(CE_WARN,
2084 			    "Memory DR disabled: invalid DR map base: 0x%p\n",
2085 			    (void *)ppvm_base);
2086 			ppvm_enable = 0;
2087 		} else if ((caddr_t)ppvm_base > ppvm_max) {
2088 			uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2089 
2090 			cmn_err(CE_WARN,
2091 			    "Memory DR disabled: insufficient DR map size:"
2092 			    " 0x%lx (needed 0x%lx)\n",
2093 			    ppvm_size - diff, ppvm_size);
2094 			ppvm_enable = 0;
2095 		}
2096 		PRM_DEBUG(ppvm_size);
2097 		PRM_DEBUG(ppvm_base);
2098 	}
2099 
2100 	/*
2101 	 * Now create generic mapping segment.  This mapping
2102 	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2103 	 * virtual address is greater than the amount of free
2104 	 * memory that is available, then we trim back the
2105 	 * segment size to that amount
2106 	 */
2107 	va = (caddr_t)SEGMAPBASE;
2108 
2109 	/*
2110 	 * 1201049: segkmap base address must be MAXBSIZE aligned
2111 	 */
2112 	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2113 
2114 	/*
2115 	 * Set size of segmap to percentage of freemem at boot,
2116 	 * but stay within the allowable range
2117 	 * Note we take percentage  before converting from pages
2118 	 * to bytes to avoid an overflow on 32-bit kernels.
2119 	 */
2120 	i = mmu_ptob((freemem * segmap_percent) / 100);
2121 
2122 	if (i < MINMAPSIZE)
2123 		i = MINMAPSIZE;
2124 
2125 	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2126 		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2127 
2128 	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
2129 
2130 	rw_enter(&kas.a_lock, RW_WRITER);
2131 	if (seg_attach(&kas, va, i, segkmap) < 0)
2132 		cmn_err(CE_PANIC, "cannot attach segkmap");
2133 
2134 	a.prot = PROT_READ | PROT_WRITE;
2135 	a.shmsize = shm_alignment;
2136 	a.nfreelist = 0;	/* use segmap driver defaults */
2137 
2138 	if (segmap_create(segkmap, (caddr_t)&a) != 0)
2139 		panic("segmap_create segkmap");
2140 	rw_exit(&kas.a_lock);
2141 
2142 	segdev_init();
2143 }
2144 
2145 static void
startup_end(void)2146 startup_end(void)
2147 {
2148 	if ((caddr_t)memlist > (caddr_t)memlist_end)
2149 		panic("memlist overflow 2");
2150 	memlist_free_block((caddr_t)memlist,
2151 	    ((caddr_t)memlist_end - (caddr_t)memlist));
2152 	memlist = NULL;
2153 
2154 	/* enable page_relocation since OBP is now done */
2155 	page_relocate_ready = 1;
2156 
2157 	/*
2158 	 * Perform tasks that get done after most of the VM
2159 	 * initialization has been done but before the clock
2160 	 * and other devices get started.
2161 	 */
2162 	kern_setup1();
2163 
2164 	/*
2165 	 * Perform CPC initialization for this CPU.
2166 	 */
2167 	kcpc_hw_init();
2168 
2169 	/*
2170 	 * Intialize the VM arenas for allocating physically
2171 	 * contiguus memory chunk for interrupt queues snd
2172 	 * allocate/register boot cpu's queues, if any and
2173 	 * allocate dump buffer for sun4v systems to store
2174 	 * extra crash information during crash dump
2175 	 */
2176 	contig_mem_init();
2177 	mach_descrip_init();
2178 
2179 	if (cpu_intrq_setup(CPU)) {
2180 		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2181 	}
2182 	cpu_intrq_register(CPU);
2183 	mach_htraptrace_setup(CPU->cpu_id);
2184 	mach_htraptrace_configure(CPU->cpu_id);
2185 	mach_dump_buffer_init();
2186 
2187 	/*
2188 	 * Initialize interrupt related stuff
2189 	 */
2190 	cpu_intr_alloc(CPU, NINTR_THREADS);
2191 
2192 	(void) splzs();			/* allow hi clock ints but not zs */
2193 
2194 	/*
2195 	 * Initialize errors.
2196 	 */
2197 	error_init();
2198 
2199 	/*
2200 	 * Note that we may have already used kernel bcopy before this
2201 	 * point - but if you really care about this, adb the use_hw_*
2202 	 * variables to 0 before rebooting.
2203 	 */
2204 	mach_hw_copy_limit();
2205 
2206 	/*
2207 	 * Install the "real" preemption guards before DDI services
2208 	 * are available.
2209 	 */
2210 	(void) prom_set_preprom(kern_preprom);
2211 	(void) prom_set_postprom(kern_postprom);
2212 	CPU->cpu_m.mutex_ready = 1;
2213 
2214 	/*
2215 	 * Initialize segnf (kernel support for non-faulting loads).
2216 	 */
2217 	segnf_init();
2218 
2219 	/*
2220 	 * Configure the root devinfo node.
2221 	 */
2222 	configure();		/* set up devices */
2223 	mach_cpu_halt_idle();
2224 }
2225 
2226 
2227 void
post_startup(void)2228 post_startup(void)
2229 {
2230 #ifdef	PTL1_PANIC_DEBUG
2231 	extern void init_ptl1_thread(void);
2232 #endif	/* PTL1_PANIC_DEBUG */
2233 	extern void abort_sequence_init(void);
2234 
2235 	/*
2236 	 * Set the system wide, processor-specific flags to be passed
2237 	 * to userland via the aux vector for performance hints and
2238 	 * instruction set extensions.
2239 	 */
2240 	bind_hwcap();
2241 
2242 	/*
2243 	 * Startup memory scrubber (if any)
2244 	 */
2245 	mach_memscrub();
2246 
2247 	/*
2248 	 * Allocate soft interrupt to handle abort sequence.
2249 	 */
2250 	abort_sequence_init();
2251 
2252 	/*
2253 	 * Configure the rest of the system.
2254 	 * Perform forceloading tasks for /etc/system.
2255 	 */
2256 	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2257 	/*
2258 	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2259 	 * ON4.0: This must be fixed or restated in /etc/systems.
2260 	 */
2261 	(void) modload("fs", "procfs");
2262 
2263 	/* load machine class specific drivers */
2264 	load_mach_drivers();
2265 
2266 	/* load platform specific drivers */
2267 	if (&load_platform_drivers)
2268 		load_platform_drivers();
2269 
2270 	/* load vis simulation module, if we are running w/fpu off */
2271 	if (!fpu_exists) {
2272 		if (modload("misc", "vis") == -1)
2273 			halt("Can't load vis");
2274 	}
2275 
2276 	mach_fpras();
2277 
2278 	maxmem = freemem;
2279 
2280 	pg_init();
2281 
2282 #ifdef	PTL1_PANIC_DEBUG
2283 	init_ptl1_thread();
2284 #endif	/* PTL1_PANIC_DEBUG */
2285 }
2286 
2287 #ifdef	PTL1_PANIC_DEBUG
2288 int		ptl1_panic_test = 0;
2289 int		ptl1_panic_xc_one_test = 0;
2290 int		ptl1_panic_xc_all_test = 0;
2291 int		ptl1_panic_xt_one_test = 0;
2292 int		ptl1_panic_xt_all_test = 0;
2293 kthread_id_t	ptl1_thread_p = NULL;
2294 kcondvar_t	ptl1_cv;
2295 kmutex_t	ptl1_mutex;
2296 int		ptl1_recurse_count_threshold = 0x40;
2297 int		ptl1_recurse_trap_threshold = 0x3d;
2298 extern void	ptl1_recurse(int, int);
2299 extern void	ptl1_panic_xt(int, int);
2300 
2301 /*
2302  * Called once per second by timeout() to wake up
2303  * the ptl1_panic thread to see if it should cause
2304  * a trap to the ptl1_panic() code.
2305  */
2306 /* ARGSUSED */
2307 static void
ptl1_wakeup(void * arg)2308 ptl1_wakeup(void *arg)
2309 {
2310 	mutex_enter(&ptl1_mutex);
2311 	cv_signal(&ptl1_cv);
2312 	mutex_exit(&ptl1_mutex);
2313 }
2314 
2315 /*
2316  * ptl1_panic cross call function:
2317  *     Needed because xc_one() and xc_some() can pass
2318  *	64 bit args but ptl1_recurse() expects ints.
2319  */
2320 static void
ptl1_panic_xc(void)2321 ptl1_panic_xc(void)
2322 {
2323 	ptl1_recurse(ptl1_recurse_count_threshold,
2324 	    ptl1_recurse_trap_threshold);
2325 }
2326 
2327 /*
2328  * The ptl1 thread waits for a global flag to be set
2329  * and uses the recurse thresholds to set the stack depth
2330  * to cause a ptl1_panic() directly via a call to ptl1_recurse
2331  * or indirectly via the cross call and cross trap functions.
2332  *
2333  * This is useful testing stack overflows and normal
2334  * ptl1_panic() states with a know stack frame.
2335  *
2336  * ptl1_recurse() is an asm function in ptl1_panic.s that
2337  * sets the {In, Local, Out, and Global} registers to a
2338  * know state on the stack and just prior to causing a
2339  * test ptl1_panic trap.
2340  */
2341 static void
ptl1_thread(void)2342 ptl1_thread(void)
2343 {
2344 	mutex_enter(&ptl1_mutex);
2345 	while (ptl1_thread_p) {
2346 		cpuset_t	other_cpus;
2347 		int		cpu_id;
2348 		int		my_cpu_id;
2349 		int		target_cpu_id;
2350 		int		target_found;
2351 
2352 		if (ptl1_panic_test) {
2353 			ptl1_recurse(ptl1_recurse_count_threshold,
2354 			    ptl1_recurse_trap_threshold);
2355 		}
2356 
2357 		/*
2358 		 * Find potential targets for x-call and x-trap,
2359 		 * if any exist while preempt is disabled we
2360 		 * start a ptl1_panic if requested via a
2361 		 * globals.
2362 		 */
2363 		kpreempt_disable();
2364 		my_cpu_id = CPU->cpu_id;
2365 		other_cpus = cpu_ready_set;
2366 		CPUSET_DEL(other_cpus, CPU->cpu_id);
2367 		target_found = 0;
2368 		if (!CPUSET_ISNULL(other_cpus)) {
2369 			/*
2370 			 * Pick the first one
2371 			 */
2372 			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2373 				if (cpu_id == my_cpu_id)
2374 					continue;
2375 
2376 				if (CPU_XCALL_READY(cpu_id)) {
2377 					target_cpu_id = cpu_id;
2378 					target_found = 1;
2379 					break;
2380 				}
2381 			}
2382 			ASSERT(target_found);
2383 
2384 			if (ptl1_panic_xc_one_test) {
2385 				xc_one(target_cpu_id,
2386 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2387 			}
2388 			if (ptl1_panic_xc_all_test) {
2389 				xc_some(other_cpus,
2390 				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2391 			}
2392 			if (ptl1_panic_xt_one_test) {
2393 				xt_one(target_cpu_id,
2394 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2395 			}
2396 			if (ptl1_panic_xt_all_test) {
2397 				xt_some(other_cpus,
2398 				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2399 			}
2400 		}
2401 		kpreempt_enable();
2402 		(void) timeout(ptl1_wakeup, NULL, hz);
2403 		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2404 	}
2405 	mutex_exit(&ptl1_mutex);
2406 }
2407 
2408 /*
2409  * Called during early startup to create the ptl1_thread
2410  */
2411 void
init_ptl1_thread(void)2412 init_ptl1_thread(void)
2413 {
2414 	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2415 	    &p0, TS_RUN, 0);
2416 }
2417 #endif	/* PTL1_PANIC_DEBUG */
2418 
2419 
2420 static void
memlist_new(uint64_t start,uint64_t len,struct memlist ** memlistp)2421 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2422 {
2423 	struct memlist *new;
2424 
2425 	new = *memlistp;
2426 	new->ml_address = start;
2427 	new->ml_size = len;
2428 	*memlistp = new + 1;
2429 }
2430 
2431 /*
2432  * Add to a memory list.
2433  * start = start of new memory segment
2434  * len = length of new memory segment in bytes
2435  * memlistp = pointer to array of available memory segment structures
2436  * curmemlistp = memory list to which to add segment.
2437  */
2438 static void
memlist_add(uint64_t start,uint64_t len,struct memlist ** memlistp,struct memlist ** curmemlistp)2439 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2440 	struct memlist **curmemlistp)
2441 {
2442 	struct memlist *new = *memlistp;
2443 
2444 	memlist_new(start, len, memlistp);
2445 	memlist_insert(new, curmemlistp);
2446 }
2447 
2448 static int
ndata_alloc_memseg(struct memlist * ndata,size_t avail)2449 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2450 {
2451 	int nseg;
2452 	size_t memseg_sz;
2453 	struct memseg *msp;
2454 
2455 	/*
2456 	 * The memseg list is for the chunks of physical memory that
2457 	 * will be managed by the vm system.  The number calculated is
2458 	 * a guess as boot may fragment it more when memory allocations
2459 	 * are made before kphysm_init().
2460 	 */
2461 	memseg_sz = (avail + 10) * sizeof (struct memseg);
2462 	memseg_sz = roundup(memseg_sz, PAGESIZE);
2463 	nseg = memseg_sz / sizeof (struct memseg);
2464 	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2465 	if (msp == NULL)
2466 		return (1);
2467 	PRM_DEBUG(memseg_free);
2468 
2469 	while (nseg--) {
2470 		msp->next = memseg_free;
2471 		memseg_free = msp;
2472 		msp++;
2473 	}
2474 	return (0);
2475 }
2476 
2477 /*
2478  * In the case of architectures that support dynamic addition of
2479  * memory at run-time there are two cases where memsegs need to
2480  * be initialized and added to the memseg list.
2481  * 1) memsegs that are constructed at startup.
2482  * 2) memsegs that are constructed at run-time on
2483  *    hot-plug capable architectures.
2484  * This code was originally part of the function kphysm_init().
2485  */
2486 
2487 static void
memseg_list_add(struct memseg * memsegp)2488 memseg_list_add(struct memseg *memsegp)
2489 {
2490 	struct memseg **prev_memsegp;
2491 	pgcnt_t num;
2492 
2493 	/* insert in memseg list, decreasing number of pages order */
2494 
2495 	num = MSEG_NPAGES(memsegp);
2496 
2497 	for (prev_memsegp = &memsegs; *prev_memsegp;
2498 	    prev_memsegp = &((*prev_memsegp)->next)) {
2499 		if (num > MSEG_NPAGES(*prev_memsegp))
2500 			break;
2501 	}
2502 
2503 	memsegp->next = *prev_memsegp;
2504 	*prev_memsegp = memsegp;
2505 
2506 	if (kpm_enable) {
2507 		memsegp->nextpa = (memsegp->next) ?
2508 		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2509 
2510 		if (prev_memsegp != &memsegs) {
2511 			struct memseg *msp;
2512 			msp = (struct memseg *)((caddr_t)prev_memsegp -
2513 			    offsetof(struct memseg, next));
2514 			msp->nextpa = va_to_pa(memsegp);
2515 		} else {
2516 			memsegspa = va_to_pa(memsegs);
2517 		}
2518 	}
2519 }
2520 
2521 /*
2522  * PSM add_physmem_cb(). US-II and newer processors have some
2523  * flavor of the prefetch capability implemented. We exploit
2524  * this capability for optimum performance.
2525  */
2526 #define	PREFETCH_BYTES	64
2527 
2528 void
add_physmem_cb(page_t * pp,pfn_t pnum)2529 add_physmem_cb(page_t *pp, pfn_t pnum)
2530 {
2531 	extern void	 prefetch_page_w(void *);
2532 
2533 	pp->p_pagenum = pnum;
2534 
2535 	/*
2536 	 * Prefetch one more page_t into E$. To prevent future
2537 	 * mishaps with the sizeof(page_t) changing on us, we
2538 	 * catch this on debug kernels if we can't bring in the
2539 	 * entire hpage with 2 PREFETCH_BYTES reads. See
2540 	 * also, sun4u/cpu/cpu_module.c
2541 	 */
2542 	/*LINTED*/
2543 	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2544 	prefetch_page_w((char *)pp);
2545 }
2546 
2547 /*
2548  * Find memseg with given pfn
2549  */
2550 static struct memseg *
memseg_find(pfn_t base,pfn_t * next)2551 memseg_find(pfn_t base, pfn_t *next)
2552 {
2553 	struct memseg *seg;
2554 
2555 	if (next != NULL)
2556 		*next = LONG_MAX;
2557 	for (seg = memsegs; seg != NULL; seg = seg->next) {
2558 		if (base >= seg->pages_base && base < seg->pages_end)
2559 			return (seg);
2560 		if (next != NULL && seg->pages_base > base &&
2561 		    seg->pages_base < *next)
2562 			*next = seg->pages_base;
2563 	}
2564 	return (NULL);
2565 }
2566 
2567 /*
2568  * Put page allocated by OBP on prom_ppages
2569  */
2570 static void
kphysm_erase(uint64_t addr,uint64_t len)2571 kphysm_erase(uint64_t addr, uint64_t len)
2572 {
2573 	struct page *pp;
2574 	struct memseg *seg;
2575 	pfn_t base = btop(addr), next;
2576 	pgcnt_t num = btop(len);
2577 
2578 	while (num != 0) {
2579 		pgcnt_t off, left;
2580 
2581 		seg = memseg_find(base, &next);
2582 		if (seg == NULL) {
2583 			if (next == LONG_MAX)
2584 				break;
2585 			left = MIN(next - base, num);
2586 			base += left, num -= left;
2587 			continue;
2588 		}
2589 		off = base - seg->pages_base;
2590 		pp = seg->pages + off;
2591 		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2592 		while (num != left) {
2593 			/*
2594 			 * init it, lock it, and hashin on prom_pages vp.
2595 			 *
2596 			 * Mark it as NONRELOC to let DR know the page
2597 			 * is locked long term, otherwise DR hangs when
2598 			 * trying to remove those pages.
2599 			 *
2600 			 * XXX	vnode offsets on the prom_ppages vnode
2601 			 *	are page numbers (gack) for >32 bit
2602 			 *	physical memory machines.
2603 			 */
2604 			PP_SETNORELOC(pp);
2605 			add_physmem_cb(pp, base);
2606 			if (page_trylock(pp, SE_EXCL) == 0)
2607 				cmn_err(CE_PANIC, "prom page locked");
2608 			(void) page_hashin(pp, &promvp,
2609 			    (offset_t)base, NULL);
2610 			(void) page_pp_lock(pp, 0, 1);
2611 			pp++, base++, num--;
2612 		}
2613 	}
2614 }
2615 
2616 static page_t *ppnext;
2617 static pgcnt_t ppleft;
2618 
2619 static void *kpm_ppnext;
2620 static pgcnt_t kpm_ppleft;
2621 
2622 /*
2623  * Create a memseg
2624  */
2625 static void
kphysm_memseg(uint64_t addr,uint64_t len)2626 kphysm_memseg(uint64_t addr, uint64_t len)
2627 {
2628 	pfn_t base = btop(addr);
2629 	pgcnt_t num = btop(len);
2630 	struct memseg *seg;
2631 
2632 	seg = memseg_free;
2633 	memseg_free = seg->next;
2634 	ASSERT(seg != NULL);
2635 
2636 	seg->pages = ppnext;
2637 	seg->epages = ppnext + num;
2638 	seg->pages_base = base;
2639 	seg->pages_end = base + num;
2640 	ppnext += num;
2641 	ppleft -= num;
2642 
2643 	if (kpm_enable) {
2644 		pgcnt_t kpnum = ptokpmpr(num);
2645 
2646 		if (kpnum > kpm_ppleft)
2647 			panic("kphysm_memseg: kpm_pp overflow");
2648 		seg->pagespa = va_to_pa(seg->pages);
2649 		seg->epagespa = va_to_pa(seg->epages);
2650 		seg->kpm_pbase = kpmptop(ptokpmp(base));
2651 		seg->kpm_nkpmpgs = kpnum;
2652 		/*
2653 		 * In the kpm_smallpage case, the kpm array
2654 		 * is 1-1 wrt the page array
2655 		 */
2656 		if (kpm_smallpages) {
2657 			kpm_spage_t *kpm_pp = kpm_ppnext;
2658 
2659 			kpm_ppnext = kpm_pp + kpnum;
2660 			seg->kpm_spages = kpm_pp;
2661 			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2662 		} else {
2663 			kpm_page_t *kpm_pp = kpm_ppnext;
2664 
2665 			kpm_ppnext = kpm_pp + kpnum;
2666 			seg->kpm_pages = kpm_pp;
2667 			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2668 			/* ASSERT no kpm overlaps */
2669 			ASSERT(
2670 			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2671 			ASSERT(memseg_find(
2672 			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2673 		}
2674 		kpm_ppleft -= kpnum;
2675 	}
2676 
2677 	memseg_list_add(seg);
2678 }
2679 
2680 /*
2681  * Add range to free list
2682  */
2683 void
kphysm_add(uint64_t addr,uint64_t len,int reclaim)2684 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2685 {
2686 	struct page *pp;
2687 	struct memseg *seg;
2688 	pfn_t base = btop(addr);
2689 	pgcnt_t num = btop(len);
2690 
2691 	seg = memseg_find(base, NULL);
2692 	ASSERT(seg != NULL);
2693 	pp = seg->pages + (base - seg->pages_base);
2694 
2695 	if (reclaim) {
2696 		struct page *rpp = pp;
2697 		struct page *lpp = pp + num;
2698 
2699 		/*
2700 		 * page should be locked on prom_ppages
2701 		 * unhash and unlock it
2702 		 */
2703 		while (rpp < lpp) {
2704 			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2705 			ASSERT(PP_ISNORELOC(rpp));
2706 			PP_CLRNORELOC(rpp);
2707 			page_pp_unlock(rpp, 0, 1);
2708 			page_hashout(rpp, NULL);
2709 			page_unlock(rpp);
2710 			rpp++;
2711 		}
2712 	}
2713 
2714 	/*
2715 	 * add_physmem() initializes the PSM part of the page
2716 	 * struct by calling the PSM back with add_physmem_cb().
2717 	 * In addition it coalesces pages into larger pages as
2718 	 * it initializes them.
2719 	 */
2720 	add_physmem(pp, num, base);
2721 }
2722 
2723 /*
2724  * kphysm_init() tackles the problem of initializing physical memory.
2725  */
2726 static void
kphysm_init(void)2727 kphysm_init(void)
2728 {
2729 	struct memlist *pmem;
2730 
2731 	ASSERT(page_hash != NULL && page_hashsz != 0);
2732 
2733 	ppnext = pp_base;
2734 	ppleft = npages;
2735 	kpm_ppnext = kpm_pp_base;
2736 	kpm_ppleft = kpm_npages;
2737 
2738 	/*
2739 	 * installed pages not on nopp_memlist go in memseg list
2740 	 */
2741 	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2742 
2743 	/*
2744 	 * Free the avail list
2745 	 */
2746 	for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2747 		kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2748 
2749 	/*
2750 	 * Erase pages that aren't available
2751 	 */
2752 	diff_memlists(phys_install, phys_avail, kphysm_erase);
2753 
2754 	build_pfn_hash();
2755 }
2756 
2757 /*
2758  * Kernel VM initialization.
2759  * Assumptions about kernel address space ordering:
2760  *	(1) gap (user space)
2761  *	(2) kernel text
2762  *	(3) kernel data/bss
2763  *	(4) gap
2764  *	(5) kernel data structures
2765  *	(6) gap
2766  *	(7) debugger (optional)
2767  *	(8) monitor
2768  *	(9) gap (possibly null)
2769  *	(10) dvma
2770  *	(11) devices
2771  */
2772 static void
kvm_init(void)2773 kvm_init(void)
2774 {
2775 	/*
2776 	 * Put the kernel segments in kernel address space.
2777 	 */
2778 	rw_enter(&kas.a_lock, RW_WRITER);
2779 	as_avlinit(&kas);
2780 
2781 	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2782 	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2783 	(void) segkmem_create(&ktextseg);
2784 
2785 	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2786 	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2787 	(void) segkmem_create(&ktexthole);
2788 
2789 	(void) seg_attach(&kas, (caddr_t)valloc_base,
2790 	    (size_t)(econtig32 - valloc_base), &kvalloc);
2791 	(void) segkmem_create(&kvalloc);
2792 
2793 	if (kmem64_base) {
2794 		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2795 		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2796 		(void) segkmem_create(&kmem64);
2797 	}
2798 
2799 	/*
2800 	 * We're about to map out /boot.  This is the beginning of the
2801 	 * system resource management transition. We can no longer
2802 	 * call into /boot for I/O or memory allocations.
2803 	 */
2804 	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2805 	(void) segkmem_create(&kvseg);
2806 	hblk_alloc_dynamic = 1;
2807 
2808 	/*
2809 	 * we need to preallocate pages for DR operations before enabling large
2810 	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2811 	 */
2812 	memseg_remap_init();
2813 
2814 	/* at this point we are ready to use large page heap */
2815 	segkmem_heap_lp_init();
2816 
2817 	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2818 	    &kvseg32);
2819 	(void) segkmem_create(&kvseg32);
2820 
2821 	/*
2822 	 * Create a segment for the debugger.
2823 	 */
2824 	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2825 	(void) segkmem_create(&kdebugseg);
2826 
2827 	rw_exit(&kas.a_lock);
2828 }
2829 
2830 char obp_tte_str[] =
2831 	"h# %x constant MMU_PAGESHIFT "
2832 	"h# %x constant TTE8K "
2833 	"h# %x constant SFHME_SIZE "
2834 	"h# %x constant SFHME_TTE "
2835 	"h# %x constant HMEBLK_TAG "
2836 	"h# %x constant HMEBLK_NEXT "
2837 	"h# %x constant HMEBLK_MISC "
2838 	"h# %x constant HMEBLK_HME1 "
2839 	"h# %x constant NHMENTS "
2840 	"h# %x constant HBLK_SZMASK "
2841 	"h# %x constant HBLK_RANGE_SHIFT "
2842 	"h# %x constant HMEBP_HBLK "
2843 	"h# %x constant HMEBLK_ENDPA "
2844 	"h# %x constant HMEBUCKET_SIZE "
2845 	"h# %x constant HTAG_SFMMUPSZ "
2846 	"h# %x constant HTAG_BSPAGE_SHIFT "
2847 	"h# %x constant HTAG_REHASH_SHIFT "
2848 	"h# %x constant SFMMU_INVALID_SHMERID "
2849 	"h# %x constant mmu_hashcnt "
2850 	"h# %p constant uhme_hash "
2851 	"h# %p constant khme_hash "
2852 	"h# %x constant UHMEHASH_SZ "
2853 	"h# %x constant KHMEHASH_SZ "
2854 	"h# %p constant KCONTEXT "
2855 	"h# %p constant KHATID "
2856 	"h# %x constant ASI_MEM "
2857 
2858 	": PHYS-X@ ( phys -- data ) "
2859 	"   ASI_MEM spacex@ "
2860 	"; "
2861 
2862 	": PHYS-W@ ( phys -- data ) "
2863 	"   ASI_MEM spacew@ "
2864 	"; "
2865 
2866 	": PHYS-L@ ( phys -- data ) "
2867 	"   ASI_MEM spaceL@ "
2868 	"; "
2869 
2870 	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2871 	"   3 * MMU_PAGESHIFT + "
2872 	"; "
2873 
2874 	": TTE_IS_VALID ( ttep -- flag ) "
2875 	"   PHYS-X@ 0< "
2876 	"; "
2877 
2878 	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2879 	"   dup TTE8K =  if "
2880 	"      drop HBLK_RANGE_SHIFT "
2881 	"   else "
2882 	"      TTE_PAGE_SHIFT "
2883 	"   then "
2884 	"; "
2885 
2886 	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2887 	"   tuck >> swap MMU_PAGESHIFT - << "
2888 	"; "
2889 
2890 	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2891 	"   >> over xor swap                    ( hash sfmmup ) "
2892 	"   KHATID <>  if                       ( hash ) "
2893 	"      UHMEHASH_SZ and                  ( bucket ) "
2894 	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2895 	"   else                                ( hash ) "
2896 	"      KHMEHASH_SZ and                  ( bucket ) "
2897 	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2898 	"   then                                ( hmebp ) "
2899 	"; "
2900 
2901 	": HME_HASH_TABLE_SEARCH "
2902 	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2903 	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2904 	"      dup HMEBLK_ENDPA <> if     ( sfmmup hmeblkp ) ( r: hblktag ) "
2905 	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2906 	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2907 	"		  true 	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2908 	"	     else						  "
2909 	"	     	  hmeblk_next + phys-x@ false 			  "
2910 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2911 	"	     then  						  "
2912 	"	  else							  "
2913 	"	     hmeblk_next + phys-x@ false 			  "
2914 	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2915 	"	  then 							  "
2916 	"      else							  "
2917 	"         drop 0 true 						  "
2918 	"      then  							  "
2919 	"   until r> drop 						  "
2920 	"; "
2921 
2922 	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2923 	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2924 	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2925 	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2926 	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2927 	"; "
2928 
2929 	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2930 	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2931 	"   TTE8K =  if                            ( hmeblkp addr ) "
2932 	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2933 	"   else                                   ( hmeblkp addr ) "
2934 	"      drop 0                              ( hmeblkp 0 ) "
2935 	"   then                                   ( hmeblkp hme-index ) "
2936 	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2937 	"   SFHME_TTE +                            ( ttep ) "
2938 	"; "
2939 
2940 	": unix-tte ( addr cnum -- false | tte-data true ) "
2941 	"    KCONTEXT = if                   ( addr ) "
2942 	"	KHATID                       ( addr khatid ) "
2943 	"    else                            ( addr ) "
2944 	"       drop false exit              ( false ) "
2945 	"    then "
2946 	"      ( addr khatid ) "
2947 	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2948 	"         2dup swap i HME_HASH_SHIFT  "
2949 					"( addr sfmmup sfmmup addr hmeshift ) "
2950 	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2951 	"         over i 4 pick               "
2952 				"( addr sfmmup hmebp sfmmup rehash addr ) "
2953 	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2954 	"         HME_HASH_TABLE_SEARCH       "
2955 					"( addr sfmmup { null | hmeblkp } ) "
2956 	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2957 	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2958 	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2959 	"               PHYS-X@ true          ( tte-data true ) "
2960 	"            else                     ( invalid-tte ) "
2961 	"               drop false            ( false ) "
2962 	"            then                     ( false | tte-data true ) "
2963 	"            unloop exit              ( false | tte-data true ) "
2964 	"         then                        ( addr sfmmup ) "
2965 	"      loop                           ( addr sfmmup ) "
2966 	"      2drop false                    ( false ) "
2967 	"; "
2968 ;
2969 
2970 void
create_va_to_tte(void)2971 create_va_to_tte(void)
2972 {
2973 	char *bp;
2974 	extern int khmehash_num, uhmehash_num;
2975 	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2976 
2977 #define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2978 
2979 	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2980 
2981 	/*
2982 	 * Teach obp how to parse our sw ttes.
2983 	 */
2984 	(void) sprintf(bp, obp_tte_str,
2985 	    MMU_PAGESHIFT,
2986 	    TTE8K,
2987 	    sizeof (struct sf_hment),
2988 	    OFFSET(struct sf_hment, hme_tte),
2989 	    OFFSET(struct hme_blk, hblk_tag),
2990 	    OFFSET(struct hme_blk, hblk_nextpa),
2991 	    OFFSET(struct hme_blk, hblk_misc),
2992 	    OFFSET(struct hme_blk, hblk_hme),
2993 	    NHMENTS,
2994 	    HBLK_SZMASK,
2995 	    HBLK_RANGE_SHIFT,
2996 	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2997 	    HMEBLK_ENDPA,
2998 	    sizeof (struct hmehash_bucket),
2999 	    HTAG_SFMMUPSZ,
3000 	    HTAG_BSPAGE_SHIFT,
3001 	    HTAG_REHASH_SHIFT,
3002 	    SFMMU_INVALID_SHMERID,
3003 	    mmu_hashcnt,
3004 	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
3005 	    (caddr_t)va_to_pa((caddr_t)khme_hash),
3006 	    UHMEHASH_SZ,
3007 	    KHMEHASH_SZ,
3008 	    KCONTEXT,
3009 	    KHATID,
3010 	    ASI_MEM);
3011 	prom_interpret(bp, 0, 0, 0, 0, 0);
3012 
3013 	kobj_free(bp, MMU_PAGESIZE);
3014 }
3015 
3016 void
install_va_to_tte(void)3017 install_va_to_tte(void)
3018 {
3019 	/*
3020 	 * advise prom that he can use unix-tte
3021 	 */
3022 	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
3023 }
3024 
3025 /*
3026  * Here we add "device-type=console" for /os-io node, for currently
3027  * our kernel console output only supports displaying text and
3028  * performing cursor-positioning operations (through kernel framebuffer
3029  * driver) and it doesn't support other functionalities required for a
3030  * standard "display" device as specified in 1275 spec. The main missing
3031  * interface defined by the 1275 spec is "draw-logo".
3032  * also see the comments above prom_stdout_is_framebuffer().
3033  */
3034 static char *create_node =
3035 	"\" /\" find-device "
3036 	"new-device "
3037 	"\" os-io\" device-name "
3038 	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
3039 	": cb-r/w  ( adr,len method$ -- #read/#written ) "
3040 	"   2>r swap 2 2r> ['] $callback  catch  if "
3041 	"      2drop 3drop 0 "
3042 	"   then "
3043 	"; "
3044 	": read ( adr,len -- #read ) "
3045 	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
3046 	"       ( retN ... ret1 N ) "
3047 	"       ?dup  if "
3048 	"               swap >r 1-  0  ?do  drop  loop  r> "
3049 	"       else "
3050 	"               -2 "
3051 	"       then "
3052 	";    "
3053 	": write ( adr,len -- #written ) "
3054 	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
3055 	"       ( retN ... ret1 N ) "
3056 	"       ?dup  if "
3057 	"               swap >r 1-  0  ?do  drop  loop  r> "
3058 	"        else "
3059 	"               0 "
3060 	"       then "
3061 	"; "
3062 	": poll-tty ( -- ) ; "
3063 	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
3064 	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
3065 	": cb-give/take ( $method -- ) "
3066 	"       0 -rot ['] $callback catch  ?dup  if "
3067 	"               >r 2drop 2drop r> throw "
3068 	"       else "
3069 	"               0  ?do  drop  loop "
3070 	"       then "
3071 	"; "
3072 	": give ( -- )  \" exit-input\" cb-give/take ; "
3073 	": take ( -- )  \" enter-input\" cb-give/take ; "
3074 	": open ( -- ok? )  true ; "
3075 	": close ( -- ) ; "
3076 	"finish-device "
3077 	"device-end ";
3078 
3079 /*
3080  * Create the OBP input/output node (FCode serial driver).
3081  * It is needed for both USB console keyboard and for
3082  * the kernel terminal emulator.  It is too early to check for a
3083  * kernel console compatible framebuffer now, so we create this
3084  * so that we're ready if we need to enable kernel terminal emulation.
3085  *
3086  * When the USB software takes over the input device at the time
3087  * consconfig runs, OBP's stdin is redirected to this node.
3088  * Whenever the FORTH user interface is used after this switch,
3089  * the node will call back into the kernel for console input.
3090  * If a serial device such as ttya or a UART with a Type 5 keyboard
3091  * attached is used, OBP takes over the serial device when the system
3092  * goes to the debugger after the system is booted.  This sharing
3093  * of the relatively simple serial device is difficult but possible.
3094  * Sharing the USB host controller is impossible due its complexity.
3095  *
3096  * Similarly to USB keyboard input redirection, after consconfig_dacf
3097  * configures a kernel console framebuffer as the standard output
3098  * device, OBP's stdout is switched to to vector through the
3099  * /os-io node into the kernel terminal emulator.
3100  */
3101 static void
startup_create_io_node(void)3102 startup_create_io_node(void)
3103 {
3104 	prom_interpret(create_node, 0, 0, 0, 0, 0);
3105 }
3106 
3107 
3108 static void
do_prom_version_check(void)3109 do_prom_version_check(void)
3110 {
3111 	int i;
3112 	pnode_t node;
3113 	char buf[64];
3114 	static char drev[] = "Down-rev firmware detected%s\n"
3115 	    "\tPlease upgrade to the following minimum version:\n"
3116 	    "\t\t%s\n";
3117 
3118 	i = prom_version_check(buf, sizeof (buf), &node);
3119 
3120 	if (i == PROM_VER64_OK)
3121 		return;
3122 
3123 	if (i == PROM_VER64_UPGRADE) {
3124 		cmn_err(CE_WARN, drev, "", buf);
3125 
3126 #ifdef	DEBUG
3127 		prom_enter_mon();	/* Type 'go' to continue */
3128 		cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3129 		return;
3130 #else
3131 		halt(0);
3132 #endif
3133 	}
3134 
3135 	/*
3136 	 * The other possibility is that this is a server running
3137 	 * good firmware, but down-rev firmware was detected on at
3138 	 * least one other cpu board. We just complain if we see
3139 	 * that.
3140 	 */
3141 	cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3142 }
3143 
3144 
3145 /*
3146  * Must be defined in platform dependent code.
3147  */
3148 extern caddr_t modtext;
3149 extern size_t modtext_sz;
3150 extern caddr_t moddata;
3151 
3152 #define	HEAPTEXT_ARENA(addr)	\
3153 	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3154 	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3155 	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3156 
3157 #define	HEAPTEXT_OVERSIZED(addr)	\
3158 	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3159 
3160 #define	HEAPTEXT_IN_NUCLEUSDATA(addr) \
3161 	(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3162 	((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3163 
3164 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3165 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3166 kmutex_t texthole_lock;
3167 
3168 char kern_bootargs[OBP_MAXPATHLEN];
3169 char kern_bootfile[OBP_MAXPATHLEN];
3170 
3171 void
kobj_vmem_init(vmem_t ** text_arena,vmem_t ** data_arena)3172 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3173 {
3174 	uintptr_t addr, limit;
3175 
3176 	addr = HEAPTEXT_BASE;
3177 	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3178 
3179 	/*
3180 	 * Before we initialize the text_arena, we want to punch holes in the
3181 	 * underlying heaptext_arena.  This guarantees that for any text
3182 	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3183 	 */
3184 	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3185 	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3186 		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3187 		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3188 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3189 	}
3190 
3191 	/*
3192 	 * Allocate one page at the oversize to break up the text region
3193 	 * from the oversized region.
3194 	 */
3195 	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3196 	    (void *)limit, (void *)(limit + PAGESIZE),
3197 	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3198 
3199 	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3200 	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3201 	    heaptext_arena, 0, VM_SLEEP);
3202 	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3203 	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3204 }
3205 
3206 caddr_t
kobj_text_alloc(vmem_t * arena,size_t size)3207 kobj_text_alloc(vmem_t *arena, size_t size)
3208 {
3209 	caddr_t rval, better;
3210 
3211 	/*
3212 	 * First, try a sleeping allocation.
3213 	 */
3214 	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3215 
3216 	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3217 		return (rval);
3218 
3219 	/*
3220 	 * We didn't get the area that we wanted.  We're going to try to do an
3221 	 * allocation with explicit constraints.
3222 	 */
3223 	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3224 	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3225 	    VM_NOSLEEP | VM_BESTFIT);
3226 
3227 	if (better != NULL) {
3228 		/*
3229 		 * That worked.  Free our first attempt and return.
3230 		 */
3231 		vmem_free(arena, rval, size);
3232 		return (better);
3233 	}
3234 
3235 	/*
3236 	 * That didn't work; we'll have to return our first attempt.
3237 	 */
3238 	return (rval);
3239 }
3240 
3241 caddr_t
kobj_texthole_alloc(caddr_t addr,size_t size)3242 kobj_texthole_alloc(caddr_t addr, size_t size)
3243 {
3244 	int arena = HEAPTEXT_ARENA(addr);
3245 	char c[30];
3246 	uintptr_t base;
3247 
3248 	if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3249 		/*
3250 		 * If this is an oversized allocation or it is allocated in
3251 		 * the nucleus data page, there is no text hole available for
3252 		 * it; return NULL.
3253 		 */
3254 		return (NULL);
3255 	}
3256 
3257 	mutex_enter(&texthole_lock);
3258 
3259 	if (texthole_arena[arena] == NULL) {
3260 		ASSERT(texthole_source[arena] == NULL);
3261 
3262 		if (arena == 0) {
3263 			texthole_source[0] = vmem_create("module_text_holesrc",
3264 			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3265 			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3266 			    0, VM_SLEEP);
3267 		} else {
3268 			base = HEAPTEXT_BASE +
3269 			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3270 
3271 			(void) snprintf(c, sizeof (c),
3272 			    "heaptext_holesrc_%d", arena);
3273 
3274 			texthole_source[arena] = vmem_create(c, (void *)base,
3275 			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3276 			    0, VM_SLEEP);
3277 		}
3278 
3279 		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3280 
3281 		texthole_arena[arena] = vmem_create(c, NULL, 0,
3282 		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3283 		    texthole_source[arena], 0, VM_SLEEP);
3284 	}
3285 
3286 	mutex_exit(&texthole_lock);
3287 
3288 	ASSERT(texthole_arena[arena] != NULL);
3289 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3290 	return (vmem_alloc(texthole_arena[arena], size,
3291 	    VM_BESTFIT | VM_NOSLEEP));
3292 }
3293 
3294 void
kobj_texthole_free(caddr_t addr,size_t size)3295 kobj_texthole_free(caddr_t addr, size_t size)
3296 {
3297 	int arena = HEAPTEXT_ARENA(addr);
3298 
3299 	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3300 	ASSERT(texthole_arena[arena] != NULL);
3301 	vmem_free(texthole_arena[arena], addr, size);
3302 }
3303 
3304 void
release_bootstrap(void)3305 release_bootstrap(void)
3306 {
3307 	if (&cif_init)
3308 		cif_init();
3309 }
3310