1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern void cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern void mach_dump_buffer_init(void); 74 extern void mach_descrip_init(void); 75 extern void mach_descrip_startup_fini(void); 76 extern void mach_memscrub(void); 77 extern void mach_fpras(void); 78 extern void mach_cpu_halt_idle(void); 79 extern void mach_hw_copy_limit(void); 80 extern void load_mach_drivers(void); 81 extern void load_tod_module(void); 82 #pragma weak load_tod_module 83 84 extern int ndata_alloc_mmfsa(struct memlist *ndata); 85 #pragma weak ndata_alloc_mmfsa 86 87 extern void cif_init(void); 88 #pragma weak cif_init 89 90 extern void parse_idprom(void); 91 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 92 extern void mem_config_init(void); 93 extern void memseg_remap_init(void); 94 95 extern void mach_kpm_init(void); 96 97 /* 98 * External Data: 99 */ 100 extern int vac_size; /* cache size in bytes */ 101 extern uint_t vac_mask; /* VAC alignment consistency mask */ 102 extern uint_t vac_colors; 103 104 /* 105 * Global Data Definitions: 106 */ 107 108 /* 109 * XXX - Don't port this to new architectures 110 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 111 * 'romp' has no use with a prom with an IEEE 1275 client interface. 112 * The driver doesn't use the value, but it depends on the symbol. 113 */ 114 void *romp; /* veritas driver won't load without romp 4154976 */ 115 /* 116 * Declare these as initialized data so we can patch them. 117 */ 118 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 119 pgcnt_t segkpsize = 120 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 121 uint_t segmap_percent = 12; /* Size of segmap segment */ 122 123 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 124 int vac_copyback = 1; 125 char *cache_mode = NULL; 126 int use_mix = 1; 127 int prom_debug = 0; 128 129 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 caddr_t sdata; /* beginning of data segment */ 151 152 caddr_t extra_etva; /* beginning of unused nucleus text */ 153 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 154 155 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 156 caddr_t nalloc_base; /* beginning of nucleus allocation */ 157 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 158 caddr_t valloc_base; /* beginning of kvalloc segment */ 159 160 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 161 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 162 163 uintptr_t shm_alignment; /* VAC address consistency modulus */ 164 struct memlist *phys_install; /* Total installed physical memory */ 165 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 166 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 167 struct memlist ndata; /* memlist of nucleus allocatable memory */ 168 int memexp_flag; /* memory expansion card flag */ 169 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 170 pgcnt_t obp_pages; /* Physical pages used by OBP */ 171 172 /* 173 * VM data structures 174 */ 175 long page_hashsz; /* Size of page hash table (power of two) */ 176 struct page *pp_base; /* Base of system page struct array */ 177 size_t pp_sz; /* Size in bytes of page struct array */ 178 struct page **page_hash; /* Page hash table */ 179 struct seg ktextseg; /* Segment used for kernel executable image */ 180 struct seg kvalloc; /* Segment used for "valloc" mapping */ 181 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 182 struct seg ktexthole; /* Segment used for nucleus text hole */ 183 struct seg kmapseg; /* Segment used for generic kernel mappings */ 184 struct seg kpmseg; /* Segment used for physical mapping */ 185 struct seg kdebugseg; /* Segment used for the kernel debugger */ 186 187 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 188 size_t kpm_pp_sz; /* Size of system kpm_page array */ 189 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 190 191 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 192 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 193 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 194 195 /* 196 * debugger pages (if allocated) 197 */ 198 struct vnode kdebugvp; 199 200 /* 201 * Segment for relocated kernel structures in 64-bit large RAM kernels 202 */ 203 struct seg kmem64; 204 205 struct memseg *memseg_base; 206 size_t memseg_sz; /* Used to translate a va to page */ 207 struct vnode unused_pages_vp; 208 209 /* 210 * VM data structures allocated early during boot. 211 */ 212 size_t pagehash_sz; 213 uint64_t memlist_sz; 214 215 char tbr_wr_addr_inited = 0; 216 217 218 /* 219 * Static Routines: 220 */ 221 static void memlist_add(uint64_t, uint64_t, struct memlist **, 222 struct memlist **); 223 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 224 pgcnt_t); 225 static void kvm_init(void); 226 227 static void startup_init(void); 228 static void startup_memlist(void); 229 static void startup_modules(void); 230 static void startup_bop_gone(void); 231 static void startup_vm(void); 232 static void startup_end(void); 233 static void setup_cage_params(void); 234 static void startup_create_io_node(void); 235 236 static pgcnt_t npages; 237 static struct memlist *memlist; 238 void *memlist_end; 239 240 static pgcnt_t bop_alloc_pages; 241 static caddr_t hblk_base; 242 uint_t hblk_alloc_dynamic = 0; 243 uint_t hblk1_min = H1MIN; 244 uint_t hblk8_min; 245 246 247 /* 248 * Hooks for unsupported platforms and down-rev firmware 249 */ 250 int iam_positron(void); 251 #pragma weak iam_positron 252 static void do_prom_version_check(void); 253 static void kpm_init(void); 254 static void kpm_npages_setup(int); 255 static void kpm_memseg_init(void); 256 257 /* 258 * After receiving a thermal interrupt, this is the number of seconds 259 * to delay before shutting off the system, assuming 260 * shutdown fails. Use /etc/system to change the delay if this isn't 261 * large enough. 262 */ 263 int thermal_powerdown_delay = 1200; 264 265 /* 266 * Used to hold off page relocations into the cage until OBP has completed 267 * its boot-time handoff of its resources to the kernel. 268 */ 269 int page_relocate_ready = 0; 270 271 /* 272 * Enable some debugging messages concerning memory usage... 273 */ 274 #ifdef DEBUGGING_MEM 275 static int debugging_mem; 276 static void 277 printmemlist(char *title, struct memlist *list) 278 { 279 if (!debugging_mem) 280 return; 281 282 printf("%s\n", title); 283 284 while (list) { 285 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 286 (uint32_t)(list->address >> 32), (uint32_t)list->address, 287 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 288 list = list->next; 289 } 290 } 291 292 void 293 printmemseg(struct memseg *memseg) 294 { 295 if (!debugging_mem) 296 return; 297 298 printf("memseg\n"); 299 300 while (memseg) { 301 prom_printf("\tpage = 0x%p, epage = 0x%p, " 302 "pfn = 0x%x, epfn = 0x%x\n", 303 memseg->pages, memseg->epages, 304 memseg->pages_base, memseg->pages_end); 305 memseg = memseg->next; 306 } 307 } 308 309 #define debug_pause(str) halt((str)) 310 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 311 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 312 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 313 #define MPRINTF3(str, a, b, c) \ 314 if (debugging_mem) prom_printf((str), (a), (b), (c)) 315 #else /* DEBUGGING_MEM */ 316 #define MPRINTF(str) 317 #define MPRINTF1(str, a) 318 #define MPRINTF2(str, a, b) 319 #define MPRINTF3(str, a, b, c) 320 #endif /* DEBUGGING_MEM */ 321 322 /* Simple message to indicate that the bootops pointer has been zeroed */ 323 #ifdef DEBUG 324 static int bootops_gone_on = 0; 325 #define BOOTOPS_GONE() \ 326 if (bootops_gone_on) \ 327 prom_printf("The bootops vec is zeroed now!\n"); 328 #else 329 #define BOOTOPS_GONE() 330 #endif /* DEBUG */ 331 332 /* 333 * Monitor pages may not be where this says they are. 334 * and the debugger may not be there either. 335 * 336 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 337 * 338 * Physical memory layout 339 * (not necessarily contiguous) 340 * (THIS IS SOMEWHAT WRONG) 341 * /-----------------------\ 342 * | monitor pages | 343 * availmem -|-----------------------| 344 * | | 345 * | page pool | 346 * | | 347 * |-----------------------| 348 * | configured tables | 349 * | buffers | 350 * firstaddr -|-----------------------| 351 * | hat data structures | 352 * |-----------------------| 353 * | kernel data, bss | 354 * |-----------------------| 355 * | interrupt stack | 356 * |-----------------------| 357 * | kernel text (RO) | 358 * |-----------------------| 359 * | trap table (4k) | 360 * |-----------------------| 361 * page 1 | panicbuf | 362 * |-----------------------| 363 * page 0 | reclaimed | 364 * |_______________________| 365 * 366 * 367 * 368 * Kernel's Virtual Memory Layout. 369 * /-----------------------\ 370 * 0xFFFFFFFF.FFFFFFFF -| |- 371 * | OBP's virtual page | 372 * | tables | 373 * 0xFFFFFFFC.00000000 -|-----------------------|- 374 * : : 375 * : : 376 * 0xFFFFFE00.00000000 -|-----------------------|- 377 * | | Ultrasparc I/II support 378 * | segkpm segment | up to 2TB of physical 379 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 380 * | | 381 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 382 * : : 383 * : : 384 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 385 * | | ^ 386 * | UltraSPARC I/II call | | 387 * | bug requires an extra | | 388 * | 4 GB of space between | | 389 * | hole and used RAM | | 390 * | | | 391 * 0xFFFFF800.00000000 -|-----------------------|- | 392 * | | | 393 * | Virtual Address Hole | UltraSPARC 394 * | on UltraSPARC I/II | I/II * ONLY * 395 * | | | 396 * 0x00000800.00000000 -|-----------------------|- | 397 * | | | 398 * | UltraSPARC I/II call | | 399 * | bug requires an extra | | 400 * | 4 GB of space between | | 401 * | hole and used RAM | | 402 * | | v 403 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 404 * : : ^ 405 * : : | 406 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 407 * | | | 408 * | 64-bit kernel ONLY | | 409 * | | | 410 * | kmem64 segment | | 411 * | | | 412 * | (Relocated extra HME | Approximately 413 * | block allocations, | 1 TB of virtual 414 * | memnode freelists, | address space 415 * | HME hash buckets, | | 416 * | mml_table, kpmp_table,| | 417 * | page_t array and | | 418 * | hashblock pool to | | 419 * | avoid hard-coded | | 420 * | 32-bit vaddr | | 421 * | limitations) | | 422 * | | v 423 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 424 * | | 425 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 426 * | | 427 * 0x00000300.00000000 -|-----------------------|- SYSBASE 428 * : : 429 * : : 430 * -|-----------------------|- 431 * | | 432 * | segmap segment | SEGMAPSIZE (1/8th physmem, 433 * | | 256G MAX) 434 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 435 * : : 436 * : : 437 * -|-----------------------|- 438 * | | 439 * | segkp | SEGKPSIZE (2GB) 440 * | | 441 * | | 442 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 443 * | | 444 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 445 * | | (SEGKPBASE - 0x400000) 446 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 447 * | | (MEMSCRUBBASE - NCARGS) 448 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 449 * | | (ARGSBASE - PPMAPSIZE) 450 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 451 * | | 452 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 453 * | | 454 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 455 * : : 456 * : : 457 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 458 * | | 459 * | OBP | 460 * | | 461 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 462 * | kmdb | 463 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 464 * : : 465 * : : 466 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 467 * | | 468 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 469 * | | ~64MB) 470 * 0x00000000.78002000 -|-----------------------| 471 * | panicbuf | 472 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 473 * : : 474 * : : 475 * | | 476 * |-----------------------|- econtig32 477 * | vm structures | 478 * 0x00000000.01C00000 |-----------------------|- nalloc_end 479 * | TSBs | 480 * |-----------------------|- end/nalloc_base 481 * | kernel data & bss | 482 * 0x00000000.01800000 -|-----------------------| 483 * : nucleus text hole : 484 * 0x00000000.01400000 -|-----------------------| 485 * : : 486 * |-----------------------| 487 * | module text | 488 * |-----------------------|- e_text/modtext 489 * | kernel text | 490 * |-----------------------| 491 * | trap table (48k) | 492 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 493 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 494 * |-----------------------| 495 * | | 496 * | invalid | 497 * | | 498 * 0x00000000.00000000 _|_______________________| 499 * 500 * 501 * 502 * 32-bit User Virtual Memory Layout. 503 * /-----------------------\ 504 * | | 505 * | invalid | 506 * | | 507 * 0xFFC00000 -|-----------------------|- USERLIMIT 508 * | user stack | 509 * : : 510 * : : 511 * : : 512 * | user data | 513 * -|-----------------------|- 514 * | user text | 515 * 0x00002000 -|-----------------------|- 516 * | invalid | 517 * 0x00000000 _|_______________________| 518 * 519 * 520 * 521 * 64-bit User Virtual Memory Layout. 522 * /-----------------------\ 523 * | | 524 * | invalid | 525 * | | 526 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 527 * | user stack | 528 * : : 529 * : : 530 * : : 531 * | user data | 532 * -|-----------------------|- 533 * | user text | 534 * 0x00000000.00100000 -|-----------------------|- 535 * | invalid | 536 * 0x00000000.00000000 _|_______________________| 537 */ 538 539 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 540 extern uint64_t ecache_flush_address(void); 541 542 #pragma weak load_platform_modules 543 #pragma weak plat_startup_memlist 544 #pragma weak ecache_init_scrub_flush_area 545 #pragma weak ecache_flush_address 546 547 548 /* 549 * By default the DR Cage is enabled for maximum OS 550 * MPSS performance. Users needing to disable the cage mechanism 551 * can set this variable to zero via /etc/system. 552 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 553 * will result in loss of DR functionality. 554 * Platforms wishing to disable kernel Cage by default 555 * should do so in their set_platform_defaults() routine. 556 */ 557 int kernel_cage_enable = 1; 558 559 static void 560 setup_cage_params(void) 561 { 562 void (*func)(void); 563 564 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 565 if (func != NULL) { 566 (*func)(); 567 return; 568 } 569 570 if (kernel_cage_enable == 0) { 571 return; 572 } 573 kcage_range_lock(); 574 if (kcage_range_init(phys_avail, 1) == 0) { 575 kcage_init(total_pages / 256); 576 } 577 kcage_range_unlock(); 578 579 if (kcage_on) { 580 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 581 } else { 582 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 583 } 584 585 } 586 587 /* 588 * Machine-dependent startup code 589 */ 590 void 591 startup(void) 592 { 593 startup_init(); 594 if (&startup_platform) 595 startup_platform(); 596 startup_memlist(); 597 startup_modules(); 598 setup_cage_params(); 599 startup_bop_gone(); 600 startup_vm(); 601 startup_end(); 602 } 603 604 struct regs sync_reg_buf; 605 uint64_t sync_tt; 606 607 void 608 sync_handler(void) 609 { 610 struct trap_info ti; 611 int i; 612 613 /* 614 * Prevent trying to talk to the other CPUs since they are 615 * sitting in the prom and won't reply. 616 */ 617 for (i = 0; i < NCPU; i++) { 618 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 619 cpu[i]->cpu_flags &= ~CPU_READY; 620 cpu[i]->cpu_flags |= CPU_QUIESCED; 621 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 622 } 623 } 624 625 /* 626 * We've managed to get here without going through the 627 * normal panic code path. Try and save some useful 628 * information. 629 */ 630 if (!panicstr && (curthread->t_panic_trap == NULL)) { 631 ti.trap_type = sync_tt; 632 ti.trap_regs = &sync_reg_buf; 633 ti.trap_addr = NULL; 634 ti.trap_mmu_fsr = 0x0; 635 636 curthread->t_panic_trap = &ti; 637 } 638 639 /* 640 * If we're re-entering the panic path, update the signature 641 * block so that the SC knows we're in the second part of panic. 642 */ 643 if (panicstr) 644 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 645 646 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 647 panic("sync initiated"); 648 } 649 650 651 static void 652 startup_init(void) 653 { 654 /* 655 * We want to save the registers while we're still in OBP 656 * so that we know they haven't been fiddled with since. 657 * (In principle, OBP can't change them just because it 658 * makes a callback, but we'd rather not depend on that 659 * behavior.) 660 */ 661 char sync_str[] = 662 "warning @ warning off : sync " 663 "%%tl-c %%tstate h# %p x! " 664 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 665 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 666 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 667 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 668 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 669 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 670 "%%y h# %p l! %%tl-c %%tt h# %p x! " 671 "sync ; warning !"; 672 673 /* 674 * 20 == num of %p substrings 675 * 16 == max num of chars %p will expand to. 676 */ 677 char bp[sizeof (sync_str) + 16 * 20]; 678 679 (void) check_boot_version(BOP_GETVERSION(bootops)); 680 681 /* 682 * Initialize ptl1 stack for the 1st CPU. 683 */ 684 ptl1_init_cpu(&cpu0); 685 686 /* 687 * Initialize the address map for cache consistent mappings 688 * to random pages; must be done after vac_size is set. 689 */ 690 ppmapinit(); 691 692 /* 693 * Initialize the PROM callback handler. 694 */ 695 init_vx_handler(); 696 697 /* 698 * have prom call sync_callback() to handle the sync and 699 * save some useful information which will be stored in the 700 * core file later. 701 */ 702 (void) sprintf((char *)bp, sync_str, 703 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 704 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 705 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 706 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 707 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 708 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 709 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 710 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 711 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 712 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 713 prom_interpret(bp, 0, 0, 0, 0, 0); 714 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 715 } 716 717 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 718 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 719 720 #define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector)) 721 722 /* 723 * As OBP takes up some RAM when the system boots, pages will already be "lost" 724 * to the system and reflected in npages by the time we see it. 725 * 726 * We only want to allocate kernel structures in the 64-bit virtual address 727 * space on systems with enough RAM to make the overhead of keeping track of 728 * an extra kernel memory segment worthwhile. 729 * 730 * Since OBP has already performed its memory allocations by this point, if we 731 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 732 * memory in the 64-bit virtual address space; otherwise keep allocations 733 * contiguous with we've mapped so far in the 32-bit virtual address space. 734 */ 735 #define MINMOVE_RAM_MB ((size_t)1900) 736 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 737 738 pgcnt_t tune_npages = (pgcnt_t) 739 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 740 741 static void 742 startup_memlist(void) 743 { 744 size_t alloc_sz; 745 size_t ctrs_sz; 746 caddr_t alloc_base; 747 caddr_t ctrs_base, ctrs_end; 748 caddr_t memspace; 749 caddr_t va; 750 int memblocks = 0; 751 struct memlist *cur; 752 size_t syslimit = (size_t)SYSLIMIT; 753 size_t sysbase = (size_t)SYSBASE; 754 int alloc_alignsize = MMU_PAGESIZE; 755 extern void page_coloring_init(void); 756 757 /* 758 * Initialize enough of the system to allow kmem_alloc to work by 759 * calling boot to allocate its memory until the time that 760 * kvm_init is completed. The page structs are allocated after 761 * rounding up end to the nearest page boundary; the memsegs are 762 * initialized and the space they use comes from the kernel heap. 763 * With appropriate initialization, they can be reallocated later 764 * to a size appropriate for the machine's configuration. 765 * 766 * At this point, memory is allocated for things that will never 767 * need to be freed, this used to be "valloced". This allows a 768 * savings as the pages don't need page structures to describe 769 * them because them will not be managed by the vm system. 770 */ 771 772 /* 773 * We're loaded by boot with the following configuration (as 774 * specified in the sun4u/conf/Mapfile): 775 * 776 * text: 4 MB chunk aligned on a 4MB boundary 777 * data & bss: 4 MB chunk aligned on a 4MB boundary 778 * 779 * These two chunks will eventually be mapped by 2 locked 4MB 780 * ttes and will represent the nucleus of the kernel. This gives 781 * us some free space that is already allocated, some or all of 782 * which is made available to kernel module text. 783 * 784 * The free space in the data-bss chunk is used for nucleus 785 * allocatable data structures and we reserve it using the 786 * nalloc_base and nalloc_end variables. This space is currently 787 * being used for hat data structures required for tlb miss 788 * handling operations. We align nalloc_base to a l2 cache 789 * linesize because this is the line size the hardware uses to 790 * maintain cache coherency. 791 * 256K is carved out for module data. 792 */ 793 794 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 795 moddata = nalloc_base; 796 e_moddata = nalloc_base + MODDATA; 797 nalloc_base = e_moddata; 798 799 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 800 valloc_base = nalloc_base; 801 802 /* 803 * Calculate the start of the data segment. 804 */ 805 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 806 807 PRM_DEBUG(moddata); 808 PRM_DEBUG(nalloc_base); 809 PRM_DEBUG(nalloc_end); 810 PRM_DEBUG(sdata); 811 812 /* 813 * Remember any slop after e_text so we can give it to the modules. 814 */ 815 PRM_DEBUG(e_text); 816 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 817 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 818 panic("nucleus text overflow"); 819 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 820 modtext; 821 PRM_DEBUG(modtext); 822 PRM_DEBUG(modtext_sz); 823 824 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 825 &boot_physavail, &boot_physavail_len, 826 &boot_virtavail, &boot_virtavail_len); 827 /* 828 * Remember what the physically available highest page is 829 * so that dumpsys works properly, and find out how much 830 * memory is installed. 831 */ 832 installed_top_size_memlist_array(boot_physinstalled, 833 boot_physinstalled_len, &physmax, &physinstalled); 834 PRM_DEBUG(physinstalled); 835 PRM_DEBUG(physmax); 836 837 /* Fill out memory nodes config structure */ 838 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 839 840 /* 841 * Get the list of physically available memory to size 842 * the number of page structures needed. 843 */ 844 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 845 /* 846 * This first snap shot of npages can represent the pages used 847 * by OBP's text and data approximately. This is used in the 848 * the calculation of the kernel size 849 */ 850 obp_pages = physinstalled - npages; 851 852 853 /* 854 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 855 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 856 * 2MB) and any excess pages are put on physavail. The assumption is 857 * that small-memory systems will need more pages more than they'll 858 * need efficiently-mapped module texts. 859 */ 860 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 861 modtext_sz > MODTEXT_SM_CAP) { 862 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 863 modtext_sz = MODTEXT_SM_CAP; 864 } else 865 extra_etpg = 0; 866 PRM_DEBUG(extra_etpg); 867 PRM_DEBUG(modtext_sz); 868 extra_etva = modtext + modtext_sz; 869 PRM_DEBUG(extra_etva); 870 871 /* 872 * Account for any pages after e_text and e_data. 873 */ 874 npages += extra_etpg; 875 npages += mmu_btopr(nalloc_end - nalloc_base); 876 PRM_DEBUG(npages); 877 878 /* 879 * npages is the maximum of available physical memory possible. 880 * (ie. it will never be more than this) 881 */ 882 883 /* 884 * initialize the nucleus memory allocator. 885 */ 886 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 887 888 /* 889 * Allocate mmu fault status area from the nucleus data area. 890 */ 891 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 892 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 893 894 /* 895 * Allocate kernel TSBs from the nucleus data area. 896 */ 897 if (ndata_alloc_tsbs(&ndata, npages) != 0) 898 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 899 900 /* 901 * Allocate cpus structs from the nucleus data area. 902 */ 903 if (ndata_alloc_cpus(&ndata) != 0) 904 cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc"); 905 906 /* 907 * Allocate dmv dispatch table from the nucleus data area. 908 */ 909 if (ndata_alloc_dmv(&ndata) != 0) 910 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 911 912 913 page_coloring_init(); 914 915 /* 916 * Allocate page_freelists bin headers for memnode 0 from the 917 * nucleus data area. 918 */ 919 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 920 cmn_err(CE_PANIC, 921 "no more nucleus memory after page free lists alloc"); 922 923 if (kpm_enable) { 924 kpm_init(); 925 /* 926 * kpm page space -- Update kpm_npages and make the 927 * same assumption about fragmenting as it is done 928 * for memseg_sz. 929 */ 930 kpm_npages_setup(memblocks + 4); 931 } 932 933 /* 934 * Allocate hat related structs from the nucleus data area. 935 */ 936 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 937 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 938 939 /* 940 * We want to do the BOP_ALLOCs before the real allocation of page 941 * structs in order to not have to allocate page structs for this 942 * memory. We need to calculate a virtual address because we want 943 * the page structs to come before other allocations in virtual address 944 * space. This is so some (if not all) of page structs can actually 945 * live in the nucleus. 946 */ 947 948 /* 949 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 950 * 951 * There are comments all over the SFMMU code warning of dire 952 * consequences if the TSBs are moved out of 32-bit space. This 953 * is largely because the asm code uses "sethi %hi(addr)"-type 954 * instructions which will not provide the expected result if the 955 * address is a 64-bit one. 956 * 957 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 958 */ 959 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 960 alloc_base = sfmmu_ktsb_alloc(alloc_base); 961 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 962 PRM_DEBUG(alloc_base); 963 964 /* 965 * Allocate IOMMU TSB array. We do this here so that the physical 966 * memory gets deducted from the PROM's physical memory list. 967 */ 968 alloc_base = iommu_tsb_init(alloc_base); 969 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 970 ecache_alignsize); 971 PRM_DEBUG(alloc_base); 972 973 /* 974 * Platforms like Starcat and OPL need special structures assigned in 975 * 32-bit virtual address space because their probing routines execute 976 * FCode, and FCode can't handle 64-bit virtual addresses... 977 */ 978 if (&plat_startup_memlist) { 979 alloc_base = plat_startup_memlist(alloc_base); 980 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 981 ecache_alignsize); 982 PRM_DEBUG(alloc_base); 983 } 984 985 /* 986 * If we have enough memory, use 4M pages for alignment because it 987 * greatly reduces the number of TLB misses we take albeit at the cost 988 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 989 * allocation.) Still, the speedup on large memory systems (e.g. > 64 990 * GB) is quite noticeable, so it is worth the effort to do if we can. 991 * 992 * Note, however, that this speedup will only occur if the boot PROM 993 * uses the largest possible MMU page size possible to map memory 994 * requests that are properly aligned and sized (for example, a request 995 * for a multiple of 4MB of memory aligned to a 4MB boundary will 996 * result in a mapping using a 4MB MMU page.) 997 * 998 * Even then, the large page mappings will only speed things up until 999 * the startup process proceeds a bit further, as when 1000 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 1001 * kernel it remaps everything but the TSBs using 8K pages anyway... 1002 * 1003 * At some point in the future, sfmmu_map_prom_mappings() will be 1004 * rewritten to copy memory mappings to the kernel using the same MMU 1005 * page sizes the PROM used. When that occurs, if the PROM did use 1006 * large MMU pages to map memory, the alignment/sizing work we're 1007 * doing now should give us a nice extra performance boost, albeit at 1008 * the cost of greater RAM usage... 1009 */ 1010 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1011 MMU_PAGESIZE); 1012 1013 PRM_DEBUG(tune_npages); 1014 PRM_DEBUG(alloc_alignsize); 1015 1016 /* 1017 * Save off where the contiguous allocations to date have ended 1018 * in econtig32. 1019 */ 1020 econtig32 = alloc_base; 1021 PRM_DEBUG(econtig32); 1022 1023 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1024 cmn_err(CE_PANIC, "econtig32 too big"); 1025 1026 /* 1027 * To avoid memory allocation collisions in the 32-bit virtual address 1028 * space, make allocations from this point forward in 64-bit virtual 1029 * address space starting at syslimit and working up. Also use the 1030 * alignment specified by alloc_alignsize, as we may be able to save 1031 * ourselves TLB misses by using larger page sizes if they're 1032 * available. 1033 * 1034 * All this is needed because on large memory systems, the default 1035 * Solaris allocations will collide with SYSBASE32, which is hard 1036 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1037 * kernels, move the allocations to a location in the 64-bit virtual 1038 * address space space, allowing those structures to grow without 1039 * worry. 1040 * 1041 * On current CPUs we'll run out of physical memory address bits before 1042 * we need to worry about the allocations running into anything else in 1043 * VM or the virtual address holes on US-I and II, as there's currently 1044 * about 1 TB of addressable space before the US-I/II VA hole. 1045 */ 1046 kmem64_base = (caddr_t)syslimit; 1047 PRM_DEBUG(kmem64_base); 1048 1049 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1050 1051 /* 1052 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1053 * them here. 1054 */ 1055 if (khme_hash == NULL || uhme_hash == NULL) { 1056 /* 1057 * alloc_hme_buckets() will align alloc_base properly before 1058 * assigning the hash buckets, so we don't need to do it 1059 * before the call... 1060 */ 1061 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1062 1063 PRM_DEBUG(alloc_base); 1064 PRM_DEBUG(khme_hash); 1065 PRM_DEBUG(uhme_hash); 1066 } 1067 1068 /* 1069 * Allocate the remaining page freelists. NUMA systems can 1070 * have lots of page freelists, one per node, which quickly 1071 * outgrow the amount of nucleus memory available. 1072 */ 1073 if (max_mem_nodes > 1) { 1074 int mnode; 1075 caddr_t alloc_start = alloc_base; 1076 1077 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1078 alloc_base = alloc_page_freelists(mnode, alloc_base, 1079 ecache_alignsize); 1080 } 1081 1082 if (alloc_base > alloc_start) { 1083 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1084 alloc_alignsize); 1085 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1086 alloc_base - alloc_start, 1087 alloc_alignsize) != alloc_start) 1088 cmn_err(CE_PANIC, 1089 "Unable to alloc page freelists\n"); 1090 } 1091 1092 PRM_DEBUG(alloc_base); 1093 } 1094 1095 if (!mml_table) { 1096 size_t mmltable_sz; 1097 1098 /* 1099 * We need to allocate the mml_table here because there 1100 * was not enough space within the nucleus. 1101 */ 1102 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1103 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1104 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1105 alloc_alignsize); 1106 1107 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1108 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1109 panic("mml_table alloc failure"); 1110 1111 alloc_base += alloc_sz; 1112 PRM_DEBUG(mml_table); 1113 PRM_DEBUG(alloc_base); 1114 } 1115 1116 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1117 size_t kpmptable_sz; 1118 caddr_t table; 1119 1120 /* 1121 * We need to allocate either kpmp_table or kpmp_stable here 1122 * because there was not enough space within the nucleus. 1123 */ 1124 kpmptable_sz = (kpm_smallpages == 0) ? 1125 sizeof (kpm_hlk_t) * kpmp_table_sz : 1126 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1127 1128 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1129 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1130 alloc_alignsize); 1131 1132 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1133 alloc_alignsize); 1134 1135 if (table != alloc_base) 1136 panic("kpmp_table or kpmp_stable alloc failure"); 1137 1138 if (kpm_smallpages == 0) { 1139 kpmp_table = (kpm_hlk_t *)table; 1140 PRM_DEBUG(kpmp_table); 1141 } else { 1142 kpmp_stable = (kpm_shlk_t *)table; 1143 PRM_DEBUG(kpmp_stable); 1144 } 1145 1146 alloc_base += alloc_sz; 1147 PRM_DEBUG(alloc_base); 1148 } 1149 1150 if (&ecache_init_scrub_flush_area) { 1151 /* 1152 * Pass alloc_base directly, as the routine itself is 1153 * responsible for any special alignment requirements... 1154 */ 1155 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1156 PRM_DEBUG(alloc_base); 1157 } 1158 1159 /* 1160 * Take the most current snapshot we can by calling mem-update. 1161 */ 1162 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1163 &boot_physavail, &boot_physavail_len, 1164 &boot_virtavail, &boot_virtavail_len); 1165 1166 /* 1167 * Reset npages and memblocks based on boot_physavail list. 1168 */ 1169 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1170 PRM_DEBUG(npages); 1171 1172 /* 1173 * Account for extra memory after e_text. 1174 */ 1175 npages += extra_etpg; 1176 1177 /* 1178 * Calculate the largest free memory chunk in the nucleus data area. 1179 * We need to figure out if page structs can fit in there or not. 1180 * We also make sure enough page structs get created for any physical 1181 * memory we might be returning to the system. 1182 */ 1183 ndata_remain_sz = ndata_maxsize(&ndata); 1184 PRM_DEBUG(ndata_remain_sz); 1185 1186 pp_sz = sizeof (struct page) * npages; 1187 1188 /* 1189 * Here's a nice bit of code based on somewhat recursive logic: 1190 * 1191 * If the page array would fit within the nucleus, we want to 1192 * add npages to cover any extra memory we may be returning back 1193 * to the system. 1194 * 1195 * HOWEVER, the page array is sized by calculating the size of 1196 * (struct page * npages), as are the pagehash table, ctrs and 1197 * memseg_list, so the very act of performing the calculation below may 1198 * in fact make the array large enough that it no longer fits in the 1199 * nucleus, meaning there would now be a much larger area of the 1200 * nucleus free that should really be added to npages, which would 1201 * make the page array that much larger, and so on. 1202 * 1203 * This also ignores the memory possibly used in the nucleus for the 1204 * the page hash, ctrs and memseg list and the fact that whether they 1205 * fit there or not varies with the npages calculation below, but we 1206 * don't even factor them into the equation at this point; perhaps we 1207 * should or perhaps we should just take the approach that the few 1208 * extra pages we could add via this calculation REALLY aren't worth 1209 * the hassle... 1210 */ 1211 if (ndata_remain_sz > pp_sz) { 1212 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1213 1214 npages += mmu_btop(spare); 1215 1216 pp_sz = npages * sizeof (struct page); 1217 1218 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1219 } 1220 1221 /* 1222 * If physmem is patched to be non-zero, use it instead of 1223 * the monitor value unless physmem is larger than the total 1224 * amount of memory on hand. 1225 */ 1226 if (physmem == 0 || physmem > npages) 1227 physmem = npages; 1228 1229 /* 1230 * If pp_base is NULL that means the routines above have determined 1231 * the page array will not fit in the nucleus; we'll have to 1232 * BOP_ALLOC() ourselves some space for them. 1233 */ 1234 if (pp_base == NULL) { 1235 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1236 alloc_alignsize); 1237 1238 alloc_sz = roundup(pp_sz, alloc_alignsize); 1239 1240 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1241 alloc_base, alloc_sz, alloc_alignsize)) != 1242 (struct page *)alloc_base) 1243 panic("page alloc failure"); 1244 1245 alloc_base += alloc_sz; 1246 } 1247 1248 /* 1249 * The page structure hash table size is a power of 2 1250 * such that the average hash chain length is PAGE_HASHAVELEN. 1251 */ 1252 page_hashsz = npages / PAGE_HASHAVELEN; 1253 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1254 pagehash_sz = sizeof (struct page *) * page_hashsz; 1255 1256 /* 1257 * We want to TRY to fit the page structure hash table, 1258 * the page size free list counters, the memseg list and 1259 * and the kpm page space in the nucleus if possible. 1260 * 1261 * alloc_sz counts how much memory needs to be allocated by 1262 * BOP_ALLOC(). 1263 */ 1264 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1265 1266 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1267 1268 /* 1269 * Size up per page size free list counters. 1270 */ 1271 ctrs_sz = page_ctrs_sz(); 1272 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1273 1274 if (ctrs_base == NULL) 1275 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1276 1277 /* 1278 * The memseg list is for the chunks of physical memory that 1279 * will be managed by the vm system. The number calculated is 1280 * a guess as boot may fragment it more when memory allocations 1281 * are made before kphysm_init(). Currently, there are two 1282 * allocations before then, so we assume each causes fragmen- 1283 * tation, and add a couple more for good measure. 1284 */ 1285 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1286 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1287 1288 if (memseg_base == NULL) 1289 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1290 1291 1292 if (kpm_enable) { 1293 /* 1294 * kpm page space -- Update kpm_npages and make the 1295 * same assumption about fragmenting as it is done 1296 * for memseg_sz above. 1297 */ 1298 kpm_npages_setup(memblocks + 4); 1299 kpm_pp_sz = (kpm_smallpages == 0) ? 1300 kpm_npages * sizeof (kpm_page_t): 1301 kpm_npages * sizeof (kpm_spage_t); 1302 1303 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1304 ecache_alignsize); 1305 1306 if (kpm_pp_base == NULL) 1307 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1308 kpm_pp_sz; 1309 } 1310 1311 if (alloc_sz > 0) { 1312 uintptr_t bop_base; 1313 1314 /* 1315 * We need extra memory allocated through BOP_ALLOC. 1316 */ 1317 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1318 alloc_alignsize); 1319 1320 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1321 1322 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1323 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1324 panic("system page struct alloc failure"); 1325 1326 alloc_base += alloc_sz; 1327 1328 if (page_hash == NULL) { 1329 page_hash = (struct page **)bop_base; 1330 bop_base = roundup(bop_base + pagehash_sz, 1331 ecache_alignsize); 1332 } 1333 1334 if (ctrs_base == NULL) { 1335 ctrs_base = (caddr_t)bop_base; 1336 bop_base = roundup(bop_base + ctrs_sz, 1337 ecache_alignsize); 1338 } 1339 1340 if (memseg_base == NULL) { 1341 memseg_base = (struct memseg *)bop_base; 1342 bop_base = roundup(bop_base + memseg_sz, 1343 ecache_alignsize); 1344 } 1345 1346 if (kpm_enable && kpm_pp_base == NULL) { 1347 kpm_pp_base = (uintptr_t)bop_base; 1348 bop_base = roundup(bop_base + kpm_pp_sz, 1349 ecache_alignsize); 1350 } 1351 1352 ASSERT(bop_base <= (uintptr_t)alloc_base); 1353 } 1354 1355 /* 1356 * Initialize per page size free list counters. 1357 */ 1358 ctrs_end = page_ctrs_alloc(ctrs_base); 1359 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1360 1361 PRM_DEBUG(page_hash); 1362 PRM_DEBUG(memseg_base); 1363 PRM_DEBUG(kpm_pp_base); 1364 PRM_DEBUG(kpm_pp_sz); 1365 PRM_DEBUG(pp_base); 1366 PRM_DEBUG(pp_sz); 1367 PRM_DEBUG(alloc_base); 1368 1369 #ifdef TRAPTRACE 1370 /* 1371 * Allocate trap trace buffer last so as not to affect 1372 * the 4M alignments of the allocations above on V9 SPARCs... 1373 */ 1374 alloc_base = trap_trace_alloc(alloc_base); 1375 PRM_DEBUG(alloc_base); 1376 #endif /* TRAPTRACE */ 1377 1378 if (kmem64_base) { 1379 /* 1380 * Set the end of the kmem64 segment for V9 SPARCs, if 1381 * appropriate... 1382 */ 1383 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1384 alloc_alignsize); 1385 1386 PRM_DEBUG(kmem64_base); 1387 PRM_DEBUG(kmem64_end); 1388 } 1389 1390 /* 1391 * Allocate space for the interrupt vector table. 1392 */ 1393 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector, 1394 IVSIZE, MMU_PAGESIZE); 1395 if (memspace != (caddr_t)intr_vector) 1396 panic("interrupt table allocation failure"); 1397 1398 /* 1399 * The memory lists from boot are allocated from the heap arena 1400 * so that later they can be freed and/or reallocated. 1401 */ 1402 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1403 panic("could not retrieve property \"extent\""); 1404 1405 /* 1406 * Between now and when we finish copying in the memory lists, 1407 * allocations happen so the space gets fragmented and the 1408 * lists longer. Leave enough space for lists twice as long 1409 * as what boot says it has now; roundup to a pagesize. 1410 * Also add space for the final phys-avail copy in the fixup 1411 * routine. 1412 */ 1413 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1414 roundup(IVSIZE, MMU_PAGESIZE)); 1415 memlist_sz *= 4; 1416 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1417 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1418 if (memspace == NULL) 1419 halt("Boot allocation failed."); 1420 1421 memlist = (struct memlist *)memspace; 1422 memlist_end = (char *)memspace + memlist_sz; 1423 1424 PRM_DEBUG(memlist); 1425 PRM_DEBUG(memlist_end); 1426 PRM_DEBUG(sysbase); 1427 PRM_DEBUG(syslimit); 1428 1429 kernelheap_init((void *)sysbase, (void *)syslimit, 1430 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1431 1432 /* 1433 * Take the most current snapshot we can by calling mem-update. 1434 */ 1435 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1436 &boot_physavail, &boot_physavail_len, 1437 &boot_virtavail, &boot_virtavail_len); 1438 1439 /* 1440 * Remove the space used by BOP_ALLOC from the kernel heap 1441 * plus the area actually used by the OBP (if any) 1442 * ignoring virtual addresses in virt_avail, above syslimit. 1443 */ 1444 virt_avail = memlist; 1445 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1446 1447 for (cur = virt_avail; cur->next; cur = cur->next) { 1448 uint64_t range_base, range_size; 1449 1450 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1451 continue; 1452 if (range_base >= (uint64_t)syslimit) 1453 break; 1454 /* 1455 * Limit the range to end at syslimit. 1456 */ 1457 range_size = MIN(cur->next->address, 1458 (uint64_t)syslimit) - range_base; 1459 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1460 0, 0, (void *)range_base, (void *)(range_base + range_size), 1461 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1462 } 1463 1464 phys_avail = memlist; 1465 (void) copy_physavail(boot_physavail, boot_physavail_len, 1466 &memlist, 0, 0); 1467 1468 /* 1469 * Add any extra memory after e_text to the phys_avail list, as long 1470 * as there's at least a page to add. 1471 */ 1472 if (extra_etpg) 1473 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1474 &memlist, &phys_avail); 1475 1476 /* 1477 * Add any extra memory after e_data to the phys_avail list as long 1478 * as there's at least a page to add. Usually, there isn't any, 1479 * since extra HME blocks typically get allocated there first before 1480 * using RAM elsewhere. 1481 */ 1482 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1483 nalloc_base = nalloc_end; 1484 ndata_remain_sz = nalloc_end - nalloc_base; 1485 1486 if (ndata_remain_sz >= MMU_PAGESIZE) 1487 memlist_add(va_to_pa(nalloc_base), 1488 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1489 1490 PRM_DEBUG(memlist); 1491 PRM_DEBUG(memlist_sz); 1492 PRM_DEBUG(memspace); 1493 1494 if ((caddr_t)memlist > (memspace + memlist_sz)) 1495 panic("memlist overflow"); 1496 1497 PRM_DEBUG(pp_base); 1498 PRM_DEBUG(memseg_base); 1499 PRM_DEBUG(npages); 1500 1501 /* 1502 * Initialize the page structures from the memory lists. 1503 */ 1504 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1505 1506 availrmem_initial = availrmem = freemem; 1507 PRM_DEBUG(availrmem); 1508 1509 /* 1510 * Some of the locks depend on page_hashsz being set! 1511 * kmem_init() depends on this; so, keep it here. 1512 */ 1513 page_lock_init(); 1514 1515 /* 1516 * Initialize kernel memory allocator. 1517 */ 1518 kmem_init(); 1519 1520 /* 1521 * Initialize bp_mapin(). 1522 */ 1523 bp_init(shm_alignment, HAT_STRICTORDER); 1524 1525 /* 1526 * Reserve space for panicbuf and intr_vector from the 32-bit heap 1527 */ 1528 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1529 panicbuf, panicbuf + PANICBUFSIZE, 1530 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1531 1532 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1533 intr_vector, (caddr_t)intr_vector + IVSIZE, 1534 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1535 1536 mem_config_init(); 1537 } 1538 1539 static void 1540 startup_modules(void) 1541 { 1542 int proplen, nhblk1, nhblk8; 1543 size_t nhblksz; 1544 pgcnt_t hblk_pages, pages_per_hblk; 1545 size_t hme8blk_sz, hme1blk_sz; 1546 1547 /* 1548 * Log any optional messages from the boot program 1549 */ 1550 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1551 if (proplen > 0) { 1552 char *msg; 1553 size_t len = (size_t)proplen; 1554 1555 msg = kmem_zalloc(len, KM_SLEEP); 1556 (void) BOP_GETPROP(bootops, "boot-message", msg); 1557 cmn_err(CE_CONT, "?%s\n", msg); 1558 kmem_free(msg, len); 1559 } 1560 1561 /* 1562 * Let the platforms have a chance to change default 1563 * values before reading system file. 1564 */ 1565 if (&set_platform_defaults) 1566 set_platform_defaults(); 1567 1568 /* 1569 * Calculate default settings of system parameters based upon 1570 * maxusers, yet allow to be overridden via the /etc/system file. 1571 */ 1572 param_calc(0); 1573 1574 mod_setup(); 1575 1576 /* 1577 * If this is a positron, complain and halt. 1578 */ 1579 if (&iam_positron && iam_positron()) { 1580 cmn_err(CE_WARN, "This hardware platform is not supported" 1581 " by this release of Solaris.\n"); 1582 #ifdef DEBUG 1583 prom_enter_mon(); /* Type 'go' to resume */ 1584 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1585 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1586 1587 #else /* DEBUG */ 1588 halt(0); 1589 #endif /* DEBUG */ 1590 } 1591 1592 /* 1593 * If we are running firmware that isn't 64-bit ready 1594 * then complain and halt. 1595 */ 1596 do_prom_version_check(); 1597 1598 /* 1599 * Initialize system parameters 1600 */ 1601 param_init(); 1602 1603 /* 1604 * maxmem is the amount of physical memory we're playing with. 1605 */ 1606 maxmem = physmem; 1607 1608 /* Set segkp limits. */ 1609 ncbase = (caddr_t)SEGDEBUGBASE; 1610 ncend = (caddr_t)SEGDEBUGBASE; 1611 1612 /* 1613 * Initialize the hat layer. 1614 */ 1615 hat_init(); 1616 1617 /* 1618 * Initialize segment management stuff. 1619 */ 1620 seg_init(); 1621 1622 /* 1623 * Create the va>tte handler, so the prom can understand 1624 * kernel translations. The handler is installed later, just 1625 * as we are about to take over the trap table from the prom. 1626 */ 1627 create_va_to_tte(); 1628 1629 /* 1630 * Load the forthdebugger (optional) 1631 */ 1632 forthdebug_init(); 1633 1634 /* 1635 * Create OBP node for console input callbacks 1636 * if it is needed. 1637 */ 1638 startup_create_io_node(); 1639 1640 if (modloadonly("fs", "specfs") == -1) 1641 halt("Can't load specfs"); 1642 1643 if (modloadonly("fs", "devfs") == -1) 1644 halt("Can't load devfs"); 1645 1646 if (modloadonly("misc", "swapgeneric") == -1) 1647 halt("Can't load swapgeneric"); 1648 1649 (void) modloadonly("sys", "lbl_edition"); 1650 1651 dispinit(); 1652 1653 /* 1654 * Infer meanings to the members of the idprom buffer. 1655 */ 1656 parse_idprom(); 1657 1658 /* Read cluster configuration data. */ 1659 clconf_init(); 1660 1661 setup_ddi(); 1662 1663 /* 1664 * Lets take this opportunity to load the root device. 1665 */ 1666 if (loadrootmodules() != 0) 1667 debug_enter("Can't load the root filesystem"); 1668 1669 /* 1670 * Load tod driver module for the tod part found on this system. 1671 * Recompute the cpu frequency/delays based on tod as tod part 1672 * tends to keep time more accurately. 1673 */ 1674 if (&load_tod_module) 1675 load_tod_module(); 1676 1677 /* 1678 * Allow platforms to load modules which might 1679 * be needed after bootops are gone. 1680 */ 1681 if (&load_platform_modules) 1682 load_platform_modules(); 1683 1684 setcpudelay(); 1685 1686 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1687 &boot_physavail, &boot_physavail_len, 1688 &boot_virtavail, &boot_virtavail_len); 1689 1690 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1691 1692 /* 1693 * Calculation and allocation of hmeblks needed to remap 1694 * the memory allocated by PROM till now: 1695 * 1696 * (1) calculate how much virtual memory has been bop_alloc'ed. 1697 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1698 * (3) calculate number of hme8blk's needed to remap this memory 1699 * (4) calculate amount of memory that's consumed by these hme8blk's 1700 * (5) add memory calculated in steps (2) and (4) above. 1701 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1702 * (7) calculate number of hme8blk's needed to remap this memory 1703 * (8) calculate amount of memory that's consumed by these hme8blk's 1704 * (9) allocate additional hme1blk's to hold large mappings. 1705 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1706 * to remap everything with 4M mappings. 1707 * (10) account for partially used hblk8's due to non-64K aligned 1708 * PROM mapping entries. 1709 * (11) add memory calculated in steps (8), (9), and (10) above. 1710 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1711 * is not ready yet, this gets bop_alloc'ed. 1712 * (13) there will be very few bop_alloc's after this point before 1713 * trap table takes over 1714 */ 1715 1716 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1717 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1718 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1719 1720 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1721 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1722 nhblk8 = bop_alloc_pages / pages_per_hblk; 1723 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1724 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1725 bop_alloc_pages += hblk_pages; 1726 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1727 nhblk8 = bop_alloc_pages / pages_per_hblk; 1728 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1729 if (nhblk1 < hblk1_min) 1730 nhblk1 = hblk1_min; 1731 if (nhblk8 < hblk8_min) 1732 nhblk8 = hblk8_min; 1733 1734 /* 1735 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1736 * boundary, the number of hblk8's needed to map the entries in the 1737 * boot_virtavail list needs to be adjusted to take this into 1738 * consideration. Thus, we need to add additional hblk8's since it 1739 * is possible that an hblk8 will not have all 8 slots used due to 1740 * alignment constraints. Since there were boot_virtavail_len entries 1741 * in that list, we need to add that many hblk8's to the number 1742 * already calculated to make sure we don't underestimate. 1743 */ 1744 nhblk8 += boot_virtavail_len; 1745 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1746 1747 /* Allocate in pagesize chunks */ 1748 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1749 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1750 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1751 } 1752 1753 static void 1754 startup_bop_gone(void) 1755 { 1756 extern int bop_io_quiesced; 1757 1758 /* 1759 * Destroy the MD initialized at startup 1760 * The startup initializes the MD framework 1761 * using prom and BOP alloc free it now. 1762 */ 1763 mach_descrip_startup_fini(); 1764 1765 /* 1766 * Call back into boot and release boots resources. 1767 */ 1768 BOP_QUIESCE_IO(bootops); 1769 bop_io_quiesced = 1; 1770 1771 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1772 &boot_physavail, &boot_physavail_len, 1773 &boot_virtavail, &boot_virtavail_len); 1774 /* 1775 * Copy physinstalled list into kernel space. 1776 */ 1777 phys_install = memlist; 1778 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1779 1780 /* 1781 * setup physically contiguous area twice as large as the ecache. 1782 * this is used while doing displacement flush of ecaches 1783 */ 1784 if (&ecache_flush_address) { 1785 ecache_flushaddr = ecache_flush_address(); 1786 if (ecache_flushaddr == (uint64_t)-1) { 1787 cmn_err(CE_PANIC, 1788 "startup: no memory to set ecache_flushaddr"); 1789 } 1790 } 1791 1792 /* 1793 * Virtual available next. 1794 */ 1795 ASSERT(virt_avail != NULL); 1796 memlist_free_list(virt_avail); 1797 virt_avail = memlist; 1798 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1799 1800 /* 1801 * Last chance to ask our booter questions .. 1802 */ 1803 } 1804 1805 1806 /* 1807 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1808 * allocations have been performed. We can't call it in startup_bop_gone 1809 * since later operations can cause obp to allocate more memory. 1810 */ 1811 void 1812 startup_fixup_physavail(void) 1813 { 1814 struct memlist *cur; 1815 1816 /* 1817 * take the most current snapshot we can by calling mem-update 1818 */ 1819 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1820 &boot_physavail, &boot_physavail_len, 1821 &boot_virtavail, &boot_virtavail_len); 1822 1823 /* 1824 * Copy phys_avail list, again. 1825 * Both the kernel/boot and the prom have been allocating 1826 * from the original list we copied earlier. 1827 */ 1828 cur = memlist; 1829 (void) copy_physavail(boot_physavail, boot_physavail_len, 1830 &memlist, 0, 0); 1831 1832 /* 1833 * Add any extra memory after e_text we added to the phys_avail list 1834 * back to the old list. 1835 */ 1836 if (extra_etpg) 1837 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1838 &memlist, &cur); 1839 if (ndata_remain_sz >= MMU_PAGESIZE) 1840 memlist_add(va_to_pa(nalloc_base), 1841 (uint64_t)ndata_remain_sz, &memlist, &cur); 1842 1843 /* 1844 * There isn't any bounds checking on the memlist area 1845 * so ensure it hasn't overgrown. 1846 */ 1847 if ((caddr_t)memlist > (caddr_t)memlist_end) 1848 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1849 1850 /* 1851 * The kernel removes the pages that were allocated for it from 1852 * the freelist, but we now have to find any -extra- pages that 1853 * the prom has allocated for it's own book-keeping, and remove 1854 * them from the freelist too. sigh. 1855 */ 1856 fix_prom_pages(phys_avail, cur); 1857 1858 ASSERT(phys_avail != NULL); 1859 memlist_free_list(phys_avail); 1860 phys_avail = cur; 1861 1862 /* 1863 * We're done with boot. Just after this point in time, boot 1864 * gets unmapped, so we can no longer rely on its services. 1865 * Zero the bootops to indicate this fact. 1866 */ 1867 bootops = (struct bootops *)NULL; 1868 BOOTOPS_GONE(); 1869 } 1870 1871 static void 1872 startup_vm(void) 1873 { 1874 size_t i; 1875 struct segmap_crargs a; 1876 struct segkpm_crargs b; 1877 1878 uint64_t avmem; 1879 caddr_t va; 1880 pgcnt_t max_phys_segkp; 1881 int mnode; 1882 1883 extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg; 1884 1885 /* 1886 * get prom's mappings, create hments for them and switch 1887 * to the kernel context. 1888 */ 1889 hat_kern_setup(); 1890 1891 /* 1892 * Take over trap table 1893 */ 1894 setup_trap_table(); 1895 1896 /* 1897 * Install the va>tte handler, so that the prom can handle 1898 * misses and understand the kernel table layout in case 1899 * we need call into the prom. 1900 */ 1901 install_va_to_tte(); 1902 1903 /* 1904 * Set a flag to indicate that the tba has been taken over. 1905 */ 1906 tba_taken_over = 1; 1907 1908 /* initialize MMU primary context register */ 1909 mmu_init_kcontext(); 1910 1911 /* 1912 * The boot cpu can now take interrupts, x-calls, x-traps 1913 */ 1914 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1915 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1916 1917 /* 1918 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1919 */ 1920 tbr_wr_addr_inited = 1; 1921 1922 /* 1923 * Initialize VM system, and map kernel address space. 1924 */ 1925 kvm_init(); 1926 1927 /* 1928 * XXX4U: previously, we initialized and turned on 1929 * the caches at this point. But of course we have 1930 * nothing to do, as the prom has already done this 1931 * for us -- main memory must be E$able at all times. 1932 */ 1933 1934 /* 1935 * If the following is true, someone has patched 1936 * phsymem to be less than the number of pages that 1937 * the system actually has. Remove pages until system 1938 * memory is limited to the requested amount. Since we 1939 * have allocated page structures for all pages, we 1940 * correct the amount of memory we want to remove 1941 * by the size of the memory used to hold page structures 1942 * for the non-used pages. 1943 */ 1944 if (physmem < npages) { 1945 pgcnt_t diff, off; 1946 struct page *pp; 1947 struct seg kseg; 1948 1949 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1950 1951 off = 0; 1952 diff = npages - physmem; 1953 diff -= mmu_btopr(diff * sizeof (struct page)); 1954 kseg.s_as = &kas; 1955 while (diff--) { 1956 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1957 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1958 &kseg, (caddr_t)off); 1959 if (pp == NULL) 1960 cmn_err(CE_PANIC, "limited physmem too much!"); 1961 page_io_unlock(pp); 1962 page_downgrade(pp); 1963 availrmem--; 1964 off += MMU_PAGESIZE; 1965 } 1966 } 1967 1968 /* 1969 * When printing memory, show the total as physmem less 1970 * that stolen by a debugger. 1971 */ 1972 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1973 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1974 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1975 1976 avmem = (uint64_t)freemem << PAGESHIFT; 1977 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1978 1979 /* For small memory systems disable automatic large pages. */ 1980 if (physmem < auto_lpg_min_physmem) { 1981 exec_lpg_disable = 1; 1982 use_brk_lpg = 0; 1983 use_stk_lpg = 0; 1984 use_zmap_lpg = 0; 1985 } 1986 1987 /* 1988 * Perform platform specific freelist processing 1989 */ 1990 if (&plat_freelist_process) { 1991 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1992 if (mem_node_config[mnode].exists) 1993 plat_freelist_process(mnode); 1994 } 1995 1996 /* 1997 * Initialize the segkp segment type. We position it 1998 * after the configured tables and buffers (whose end 1999 * is given by econtig) and before V_WKBASE_ADDR. 2000 * Also in this area is segkmap (size SEGMAPSIZE). 2001 */ 2002 2003 /* XXX - cache alignment? */ 2004 va = (caddr_t)SEGKPBASE; 2005 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2006 2007 max_phys_segkp = (physmem * 2); 2008 2009 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2010 segkpsize = btop(SEGKPDEFSIZE); 2011 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2012 "segkpsize has been reset to %ld pages", segkpsize); 2013 } 2014 2015 i = ptob(MIN(segkpsize, max_phys_segkp)); 2016 2017 rw_enter(&kas.a_lock, RW_WRITER); 2018 if (seg_attach(&kas, va, i, segkp) < 0) 2019 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2020 if (segkp_create(segkp) != 0) 2021 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2022 rw_exit(&kas.a_lock); 2023 2024 /* 2025 * kpm segment 2026 */ 2027 segmap_kpm = kpm_enable && 2028 segmap_kpm && PAGESIZE == MAXBSIZE; 2029 2030 if (kpm_enable) { 2031 rw_enter(&kas.a_lock, RW_WRITER); 2032 2033 /* 2034 * The segkpm virtual range range is larger than the 2035 * actual physical memory size and also covers gaps in 2036 * the physical address range for the following reasons: 2037 * . keep conversion between segkpm and physical addresses 2038 * simple, cheap and unambiguous. 2039 * . avoid extension/shrink of the the segkpm in case of DR. 2040 * . avoid complexity for handling of virtual addressed 2041 * caches, segkpm and the regular mapping scheme must be 2042 * kept in sync wrt. the virtual color of mapped pages. 2043 * Any accesses to virtual segkpm ranges not backed by 2044 * physical memory will fall through the memseg pfn hash 2045 * and will be handled in segkpm_fault. 2046 * Additional kpm_size spaces needed for vac alias prevention. 2047 */ 2048 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2049 segkpm) < 0) 2050 cmn_err(CE_PANIC, "cannot attach segkpm"); 2051 2052 b.prot = PROT_READ | PROT_WRITE; 2053 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2054 2055 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2056 panic("segkpm_create segkpm"); 2057 2058 rw_exit(&kas.a_lock); 2059 2060 mach_kpm_init(); 2061 } 2062 2063 /* 2064 * Now create generic mapping segment. This mapping 2065 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2066 * virtual address is greater than the amount of free 2067 * memory that is available, then we trim back the 2068 * segment size to that amount 2069 */ 2070 va = (caddr_t)SEGMAPBASE; 2071 2072 /* 2073 * 1201049: segkmap base address must be MAXBSIZE aligned 2074 */ 2075 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2076 2077 /* 2078 * Set size of segmap to percentage of freemem at boot, 2079 * but stay within the allowable range 2080 * Note we take percentage before converting from pages 2081 * to bytes to avoid an overflow on 32-bit kernels. 2082 */ 2083 i = mmu_ptob((freemem * segmap_percent) / 100); 2084 2085 if (i < MINMAPSIZE) 2086 i = MINMAPSIZE; 2087 2088 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2089 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2090 2091 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2092 2093 rw_enter(&kas.a_lock, RW_WRITER); 2094 if (seg_attach(&kas, va, i, segkmap) < 0) 2095 cmn_err(CE_PANIC, "cannot attach segkmap"); 2096 2097 a.prot = PROT_READ | PROT_WRITE; 2098 a.shmsize = shm_alignment; 2099 a.nfreelist = 0; /* use segmap driver defaults */ 2100 2101 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2102 panic("segmap_create segkmap"); 2103 rw_exit(&kas.a_lock); 2104 2105 segdev_init(); 2106 } 2107 2108 static void 2109 startup_end(void) 2110 { 2111 if ((caddr_t)memlist > (caddr_t)memlist_end) 2112 panic("memlist overflow 2"); 2113 memlist_free_block((caddr_t)memlist, 2114 ((caddr_t)memlist_end - (caddr_t)memlist)); 2115 memlist = NULL; 2116 2117 /* enable page_relocation since OBP is now done */ 2118 page_relocate_ready = 1; 2119 2120 /* 2121 * Perform tasks that get done after most of the VM 2122 * initialization has been done but before the clock 2123 * and other devices get started. 2124 */ 2125 kern_setup1(); 2126 2127 /* 2128 * Intialize the VM arenas for allocating physically 2129 * contiguus memory chunk for interrupt queues snd 2130 * allocate/register boot cpu's queues, if any and 2131 * allocate dump buffer for sun4v systems to store 2132 * extra crash information during crash dump 2133 */ 2134 contig_mem_init(); 2135 mach_descrip_init(); 2136 cpu_intrq_setup(CPU); 2137 cpu_intrq_register(CPU); 2138 mach_htraptrace_setup(CPU->cpu_id); 2139 mach_htraptrace_configure(CPU->cpu_id); 2140 mach_dump_buffer_init(); 2141 2142 /* 2143 * Initialize interrupt related stuff 2144 */ 2145 cpu_intr_alloc(CPU, NINTR_THREADS); 2146 2147 (void) splzs(); /* allow hi clock ints but not zs */ 2148 2149 /* 2150 * Initialize errors. 2151 */ 2152 error_init(); 2153 2154 /* 2155 * Note that we may have already used kernel bcopy before this 2156 * point - but if you really care about this, adb the use_hw_* 2157 * variables to 0 before rebooting. 2158 */ 2159 mach_hw_copy_limit(); 2160 2161 /* 2162 * Install the "real" preemption guards before DDI services 2163 * are available. 2164 */ 2165 (void) prom_set_preprom(kern_preprom); 2166 (void) prom_set_postprom(kern_postprom); 2167 CPU->cpu_m.mutex_ready = 1; 2168 2169 /* 2170 * Initialize segnf (kernel support for non-faulting loads). 2171 */ 2172 segnf_init(); 2173 2174 /* 2175 * Configure the root devinfo node. 2176 */ 2177 configure(); /* set up devices */ 2178 mach_cpu_halt_idle(); 2179 } 2180 2181 2182 void 2183 post_startup(void) 2184 { 2185 #ifdef PTL1_PANIC_DEBUG 2186 extern void init_ptl1_thread(void); 2187 #endif /* PTL1_PANIC_DEBUG */ 2188 extern void abort_sequence_init(void); 2189 2190 /* 2191 * Set the system wide, processor-specific flags to be passed 2192 * to userland via the aux vector for performance hints and 2193 * instruction set extensions. 2194 */ 2195 bind_hwcap(); 2196 2197 /* 2198 * Startup memory scrubber (if any) 2199 */ 2200 mach_memscrub(); 2201 2202 /* 2203 * Allocate soft interrupt to handle abort sequence. 2204 */ 2205 abort_sequence_init(); 2206 2207 /* 2208 * Configure the rest of the system. 2209 * Perform forceloading tasks for /etc/system. 2210 */ 2211 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2212 /* 2213 * ON4.0: Force /proc module in until clock interrupt handle fixed 2214 * ON4.0: This must be fixed or restated in /etc/systems. 2215 */ 2216 (void) modload("fs", "procfs"); 2217 2218 /* load machine class specific drivers */ 2219 load_mach_drivers(); 2220 2221 /* load platform specific drivers */ 2222 if (&load_platform_drivers) 2223 load_platform_drivers(); 2224 2225 /* load vis simulation module, if we are running w/fpu off */ 2226 if (!fpu_exists) { 2227 if (modload("misc", "vis") == -1) 2228 halt("Can't load vis"); 2229 } 2230 2231 mach_fpras(); 2232 2233 maxmem = freemem; 2234 2235 #ifdef PTL1_PANIC_DEBUG 2236 init_ptl1_thread(); 2237 #endif /* PTL1_PANIC_DEBUG */ 2238 2239 if (&cif_init) 2240 cif_init(); 2241 } 2242 2243 #ifdef PTL1_PANIC_DEBUG 2244 int ptl1_panic_test = 0; 2245 int ptl1_panic_xc_one_test = 0; 2246 int ptl1_panic_xc_all_test = 0; 2247 int ptl1_panic_xt_one_test = 0; 2248 int ptl1_panic_xt_all_test = 0; 2249 kthread_id_t ptl1_thread_p = NULL; 2250 kcondvar_t ptl1_cv; 2251 kmutex_t ptl1_mutex; 2252 int ptl1_recurse_count_threshold = 0x40; 2253 int ptl1_recurse_trap_threshold = 0x3d; 2254 extern void ptl1_recurse(int, int); 2255 extern void ptl1_panic_xt(int, int); 2256 2257 /* 2258 * Called once per second by timeout() to wake up 2259 * the ptl1_panic thread to see if it should cause 2260 * a trap to the ptl1_panic() code. 2261 */ 2262 /* ARGSUSED */ 2263 static void 2264 ptl1_wakeup(void *arg) 2265 { 2266 mutex_enter(&ptl1_mutex); 2267 cv_signal(&ptl1_cv); 2268 mutex_exit(&ptl1_mutex); 2269 } 2270 2271 /* 2272 * ptl1_panic cross call function: 2273 * Needed because xc_one() and xc_some() can pass 2274 * 64 bit args but ptl1_recurse() expects ints. 2275 */ 2276 static void 2277 ptl1_panic_xc(void) 2278 { 2279 ptl1_recurse(ptl1_recurse_count_threshold, 2280 ptl1_recurse_trap_threshold); 2281 } 2282 2283 /* 2284 * The ptl1 thread waits for a global flag to be set 2285 * and uses the recurse thresholds to set the stack depth 2286 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2287 * or indirectly via the cross call and cross trap functions. 2288 * 2289 * This is useful testing stack overflows and normal 2290 * ptl1_panic() states with a know stack frame. 2291 * 2292 * ptl1_recurse() is an asm function in ptl1_panic.s that 2293 * sets the {In, Local, Out, and Global} registers to a 2294 * know state on the stack and just prior to causing a 2295 * test ptl1_panic trap. 2296 */ 2297 static void 2298 ptl1_thread(void) 2299 { 2300 mutex_enter(&ptl1_mutex); 2301 while (ptl1_thread_p) { 2302 cpuset_t other_cpus; 2303 int cpu_id; 2304 int my_cpu_id; 2305 int target_cpu_id; 2306 int target_found; 2307 2308 if (ptl1_panic_test) { 2309 ptl1_recurse(ptl1_recurse_count_threshold, 2310 ptl1_recurse_trap_threshold); 2311 } 2312 2313 /* 2314 * Find potential targets for x-call and x-trap, 2315 * if any exist while preempt is disabled we 2316 * start a ptl1_panic if requested via a 2317 * globals. 2318 */ 2319 kpreempt_disable(); 2320 my_cpu_id = CPU->cpu_id; 2321 other_cpus = cpu_ready_set; 2322 CPUSET_DEL(other_cpus, CPU->cpu_id); 2323 target_found = 0; 2324 if (!CPUSET_ISNULL(other_cpus)) { 2325 /* 2326 * Pick the first one 2327 */ 2328 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2329 if (cpu_id == my_cpu_id) 2330 continue; 2331 2332 if (CPU_XCALL_READY(cpu_id)) { 2333 target_cpu_id = cpu_id; 2334 target_found = 1; 2335 break; 2336 } 2337 } 2338 ASSERT(target_found); 2339 2340 if (ptl1_panic_xc_one_test) { 2341 xc_one(target_cpu_id, 2342 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2343 } 2344 if (ptl1_panic_xc_all_test) { 2345 xc_some(other_cpus, 2346 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2347 } 2348 if (ptl1_panic_xt_one_test) { 2349 xt_one(target_cpu_id, 2350 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2351 } 2352 if (ptl1_panic_xt_all_test) { 2353 xt_some(other_cpus, 2354 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2355 } 2356 } 2357 kpreempt_enable(); 2358 (void) timeout(ptl1_wakeup, NULL, hz); 2359 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2360 } 2361 mutex_exit(&ptl1_mutex); 2362 } 2363 2364 /* 2365 * Called during early startup to create the ptl1_thread 2366 */ 2367 void 2368 init_ptl1_thread(void) 2369 { 2370 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2371 &p0, TS_RUN, 0); 2372 } 2373 #endif /* PTL1_PANIC_DEBUG */ 2374 2375 2376 /* 2377 * Add to a memory list. 2378 * start = start of new memory segment 2379 * len = length of new memory segment in bytes 2380 * memlistp = pointer to array of available memory segment structures 2381 * curmemlistp = memory list to which to add segment. 2382 */ 2383 static void 2384 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2385 struct memlist **curmemlistp) 2386 { 2387 struct memlist *new; 2388 2389 new = *memlistp; 2390 new->address = start; 2391 new->size = len; 2392 *memlistp = new + 1; 2393 2394 memlist_insert(new, curmemlistp); 2395 } 2396 2397 /* 2398 * In the case of architectures that support dynamic addition of 2399 * memory at run-time there are two cases where memsegs need to 2400 * be initialized and added to the memseg list. 2401 * 1) memsegs that are constructed at startup. 2402 * 2) memsegs that are constructed at run-time on 2403 * hot-plug capable architectures. 2404 * This code was originally part of the function kphysm_init(). 2405 */ 2406 2407 static void 2408 memseg_list_add(struct memseg *memsegp) 2409 { 2410 struct memseg **prev_memsegp; 2411 pgcnt_t num; 2412 2413 /* insert in memseg list, decreasing number of pages order */ 2414 2415 num = MSEG_NPAGES(memsegp); 2416 2417 for (prev_memsegp = &memsegs; *prev_memsegp; 2418 prev_memsegp = &((*prev_memsegp)->next)) { 2419 if (num > MSEG_NPAGES(*prev_memsegp)) 2420 break; 2421 } 2422 2423 memsegp->next = *prev_memsegp; 2424 *prev_memsegp = memsegp; 2425 2426 if (kpm_enable) { 2427 memsegp->nextpa = (memsegp->next) ? 2428 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2429 2430 if (prev_memsegp != &memsegs) { 2431 struct memseg *msp; 2432 msp = (struct memseg *)((caddr_t)prev_memsegp - 2433 offsetof(struct memseg, next)); 2434 msp->nextpa = va_to_pa(memsegp); 2435 } else { 2436 memsegspa = va_to_pa(memsegs); 2437 } 2438 } 2439 } 2440 2441 /* 2442 * PSM add_physmem_cb(). US-II and newer processors have some 2443 * flavor of the prefetch capability implemented. We exploit 2444 * this capability for optimum performance. 2445 */ 2446 #define PREFETCH_BYTES 64 2447 2448 void 2449 add_physmem_cb(page_t *pp, pfn_t pnum) 2450 { 2451 extern void prefetch_page_w(void *); 2452 2453 pp->p_pagenum = pnum; 2454 2455 /* 2456 * Prefetch one more page_t into E$. To prevent future 2457 * mishaps with the sizeof(page_t) changing on us, we 2458 * catch this on debug kernels if we can't bring in the 2459 * entire hpage with 2 PREFETCH_BYTES reads. See 2460 * also, sun4u/cpu/cpu_module.c 2461 */ 2462 /*LINTED*/ 2463 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2464 prefetch_page_w((char *)pp); 2465 } 2466 2467 /* 2468 * kphysm_init() tackles the problem of initializing physical memory. 2469 * The old startup made some assumptions about the kernel living in 2470 * physically contiguous space which is no longer valid. 2471 */ 2472 static void 2473 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2474 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2475 { 2476 struct memlist *pmem; 2477 struct memseg *msp; 2478 pfn_t base; 2479 pgcnt_t num; 2480 pfn_t lastseg_pages_end = 0; 2481 pgcnt_t nelem_used = 0; 2482 2483 ASSERT(page_hash != NULL && page_hashsz != 0); 2484 2485 msp = memsegp; 2486 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2487 2488 /* 2489 * Build the memsegs entry 2490 */ 2491 num = btop(pmem->size); 2492 if (num > npages) 2493 num = npages; 2494 npages -= num; 2495 base = btop(pmem->address); 2496 2497 msp->pages = pp; 2498 msp->epages = pp + num; 2499 msp->pages_base = base; 2500 msp->pages_end = base + num; 2501 2502 if (kpm_enable) { 2503 pfn_t pbase_a; 2504 pfn_t pend_a; 2505 pfn_t prev_pend_a; 2506 pgcnt_t nelem; 2507 2508 msp->pagespa = va_to_pa(pp); 2509 msp->epagespa = va_to_pa(pp + num); 2510 pbase_a = kpmptop(ptokpmp(base)); 2511 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2512 nelem = ptokpmp(pend_a - pbase_a); 2513 msp->kpm_nkpmpgs = nelem; 2514 msp->kpm_pbase = pbase_a; 2515 if (lastseg_pages_end) { 2516 /* 2517 * Assume phys_avail is in ascending order 2518 * of physical addresses. 2519 */ 2520 ASSERT(base + num > lastseg_pages_end); 2521 prev_pend_a = kpmptop( 2522 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2523 2524 if (prev_pend_a > pbase_a) { 2525 /* 2526 * Overlap, more than one memseg may 2527 * point to the same kpm_page range. 2528 */ 2529 if (kpm_smallpages == 0) { 2530 msp->kpm_pages = 2531 (kpm_page_t *)kpm_pp - 1; 2532 kpm_pp = (uintptr_t) 2533 ((kpm_page_t *)kpm_pp 2534 + nelem - 1); 2535 } else { 2536 msp->kpm_spages = 2537 (kpm_spage_t *)kpm_pp - 1; 2538 kpm_pp = (uintptr_t) 2539 ((kpm_spage_t *)kpm_pp 2540 + nelem - 1); 2541 } 2542 nelem_used += nelem - 1; 2543 2544 } else { 2545 if (kpm_smallpages == 0) { 2546 msp->kpm_pages = 2547 (kpm_page_t *)kpm_pp; 2548 kpm_pp = (uintptr_t) 2549 ((kpm_page_t *)kpm_pp 2550 + nelem); 2551 } else { 2552 msp->kpm_spages = 2553 (kpm_spage_t *)kpm_pp; 2554 kpm_pp = (uintptr_t) 2555 ((kpm_spage_t *) 2556 kpm_pp + nelem); 2557 } 2558 nelem_used += nelem; 2559 } 2560 2561 } else { 2562 if (kpm_smallpages == 0) { 2563 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2564 kpm_pp = (uintptr_t) 2565 ((kpm_page_t *)kpm_pp + nelem); 2566 } else { 2567 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2568 kpm_pp = (uintptr_t) 2569 ((kpm_spage_t *)kpm_pp + nelem); 2570 } 2571 nelem_used = nelem; 2572 } 2573 2574 if (nelem_used > kpm_npages) 2575 panic("kphysm_init: kpm_pp overflow\n"); 2576 2577 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2578 lastseg_pages_end = msp->pages_end; 2579 } 2580 2581 memseg_list_add(msp); 2582 2583 /* 2584 * add_physmem() initializes the PSM part of the page 2585 * struct by calling the PSM back with add_physmem_cb(). 2586 * In addition it coalesces pages into larger pages as 2587 * it initializes them. 2588 */ 2589 add_physmem(pp, num, base); 2590 pp += num; 2591 msp++; 2592 } 2593 2594 build_pfn_hash(); 2595 } 2596 2597 /* 2598 * Kernel VM initialization. 2599 * Assumptions about kernel address space ordering: 2600 * (1) gap (user space) 2601 * (2) kernel text 2602 * (3) kernel data/bss 2603 * (4) gap 2604 * (5) kernel data structures 2605 * (6) gap 2606 * (7) debugger (optional) 2607 * (8) monitor 2608 * (9) gap (possibly null) 2609 * (10) dvma 2610 * (11) devices 2611 */ 2612 static void 2613 kvm_init(void) 2614 { 2615 /* 2616 * Put the kernel segments in kernel address space. 2617 */ 2618 rw_enter(&kas.a_lock, RW_WRITER); 2619 as_avlinit(&kas); 2620 2621 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2622 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2623 (void) segkmem_create(&ktextseg); 2624 2625 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2626 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2627 (void) segkmem_create(&ktexthole); 2628 2629 (void) seg_attach(&kas, (caddr_t)valloc_base, 2630 (size_t)(econtig32 - valloc_base), &kvalloc); 2631 (void) segkmem_create(&kvalloc); 2632 2633 if (kmem64_base) { 2634 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2635 (size_t)(kmem64_end - kmem64_base), &kmem64); 2636 (void) segkmem_create(&kmem64); 2637 } 2638 2639 /* 2640 * We're about to map out /boot. This is the beginning of the 2641 * system resource management transition. We can no longer 2642 * call into /boot for I/O or memory allocations. 2643 */ 2644 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2645 (void) segkmem_create(&kvseg); 2646 hblk_alloc_dynamic = 1; 2647 2648 /* 2649 * we need to preallocate pages for DR operations before enabling large 2650 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2651 */ 2652 memseg_remap_init(); 2653 2654 /* at this point we are ready to use large page heap */ 2655 segkmem_heap_lp_init(); 2656 2657 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2658 &kvseg32); 2659 (void) segkmem_create(&kvseg32); 2660 2661 /* 2662 * Create a segment for the debugger. 2663 */ 2664 (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2665 &kdebugseg); 2666 (void) segkmem_create(&kdebugseg); 2667 2668 rw_exit(&kas.a_lock); 2669 } 2670 2671 char obp_tte_str[] = 2672 "h# %x constant MMU_PAGESHIFT " 2673 "h# %x constant TTE8K " 2674 "h# %x constant SFHME_SIZE " 2675 "h# %x constant SFHME_TTE " 2676 "h# %x constant HMEBLK_TAG " 2677 "h# %x constant HMEBLK_NEXT " 2678 "h# %x constant HMEBLK_MISC " 2679 "h# %x constant HMEBLK_HME1 " 2680 "h# %x constant NHMENTS " 2681 "h# %x constant HBLK_SZMASK " 2682 "h# %x constant HBLK_RANGE_SHIFT " 2683 "h# %x constant HMEBP_HBLK " 2684 "h# %x constant HMEBUCKET_SIZE " 2685 "h# %x constant HTAG_SFMMUPSZ " 2686 "h# %x constant HTAG_REHASHSZ " 2687 "h# %x constant mmu_hashcnt " 2688 "h# %p constant uhme_hash " 2689 "h# %p constant khme_hash " 2690 "h# %x constant UHMEHASH_SZ " 2691 "h# %x constant KHMEHASH_SZ " 2692 "h# %p constant KCONTEXT " 2693 "h# %p constant KHATID " 2694 "h# %x constant ASI_MEM " 2695 2696 ": PHYS-X@ ( phys -- data ) " 2697 " ASI_MEM spacex@ " 2698 "; " 2699 2700 ": PHYS-W@ ( phys -- data ) " 2701 " ASI_MEM spacew@ " 2702 "; " 2703 2704 ": PHYS-L@ ( phys -- data ) " 2705 " ASI_MEM spaceL@ " 2706 "; " 2707 2708 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2709 " 3 * MMU_PAGESHIFT + " 2710 "; " 2711 2712 ": TTE_IS_VALID ( ttep -- flag ) " 2713 " PHYS-X@ 0< " 2714 "; " 2715 2716 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2717 " dup TTE8K = if " 2718 " drop HBLK_RANGE_SHIFT " 2719 " else " 2720 " TTE_PAGE_SHIFT " 2721 " then " 2722 "; " 2723 2724 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2725 " tuck >> swap MMU_PAGESHIFT - << " 2726 "; " 2727 2728 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2729 " >> over xor swap ( hash sfmmup ) " 2730 " KHATID <> if ( hash ) " 2731 " UHMEHASH_SZ and ( bucket ) " 2732 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2733 " else ( hash ) " 2734 " KHMEHASH_SZ and ( bucket ) " 2735 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2736 " then ( hmebp ) " 2737 "; " 2738 2739 ": HME_HASH_TABLE_SEARCH " 2740 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2741 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2742 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2743 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2744 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2745 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2746 " else " 2747 " hmeblk_next + phys-x@ false " 2748 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2749 " then " 2750 " else " 2751 " hmeblk_next + phys-x@ false " 2752 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2753 " then " 2754 " else " 2755 " true " 2756 " then " 2757 " until r> drop " 2758 "; " 2759 2760 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2761 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2762 " HTAG_REHASHSZ << or nip ( hblktag ) " 2763 "; " 2764 2765 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2766 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2767 " TTE8K = if ( hmeblkp addr ) " 2768 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2769 " else ( hmeblkp addr ) " 2770 " drop 0 ( hmeblkp 0 ) " 2771 " then ( hmeblkp hme-index ) " 2772 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2773 " SFHME_TTE + ( ttep ) " 2774 "; " 2775 2776 ": unix-tte ( addr cnum -- false | tte-data true ) " 2777 " KCONTEXT = if ( addr ) " 2778 " KHATID ( addr khatid ) " 2779 " else ( addr ) " 2780 " drop false exit ( false ) " 2781 " then " 2782 " ( addr khatid ) " 2783 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2784 " 2dup swap i HME_HASH_SHIFT " 2785 "( addr sfmmup sfmmup addr hmeshift ) " 2786 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2787 " over i 4 pick " 2788 "( addr sfmmup hmebp sfmmup rehash addr ) " 2789 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2790 " HME_HASH_TABLE_SEARCH " 2791 "( addr sfmmup { null | hmeblkp } ) " 2792 " ?dup if ( addr sfmmup hmeblkp ) " 2793 " nip swap HBLK_TO_TTEP ( ttep ) " 2794 " dup TTE_IS_VALID if ( valid-ttep ) " 2795 " PHYS-X@ true ( tte-data true ) " 2796 " else ( invalid-tte ) " 2797 " drop false ( false ) " 2798 " then ( false | tte-data true ) " 2799 " unloop exit ( false | tte-data true ) " 2800 " then ( addr sfmmup ) " 2801 " loop ( addr sfmmup ) " 2802 " 2drop false ( false ) " 2803 "; " 2804 ; 2805 2806 void 2807 create_va_to_tte(void) 2808 { 2809 char *bp; 2810 extern int khmehash_num, uhmehash_num; 2811 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2812 2813 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2814 2815 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2816 2817 /* 2818 * Teach obp how to parse our sw ttes. 2819 */ 2820 (void) sprintf(bp, obp_tte_str, 2821 MMU_PAGESHIFT, 2822 TTE8K, 2823 sizeof (struct sf_hment), 2824 OFFSET(struct sf_hment, hme_tte), 2825 OFFSET(struct hme_blk, hblk_tag), 2826 OFFSET(struct hme_blk, hblk_nextpa), 2827 OFFSET(struct hme_blk, hblk_misc), 2828 OFFSET(struct hme_blk, hblk_hme), 2829 NHMENTS, 2830 HBLK_SZMASK, 2831 HBLK_RANGE_SHIFT, 2832 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2833 sizeof (struct hmehash_bucket), 2834 HTAG_SFMMUPSZ, 2835 HTAG_REHASHSZ, 2836 mmu_hashcnt, 2837 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2838 (caddr_t)va_to_pa((caddr_t)khme_hash), 2839 UHMEHASH_SZ, 2840 KHMEHASH_SZ, 2841 KCONTEXT, 2842 KHATID, 2843 ASI_MEM); 2844 prom_interpret(bp, 0, 0, 0, 0, 0); 2845 2846 kobj_free(bp, MMU_PAGESIZE); 2847 } 2848 2849 void 2850 install_va_to_tte(void) 2851 { 2852 /* 2853 * advise prom that he can use unix-tte 2854 */ 2855 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2856 } 2857 2858 2859 /* 2860 * Because kmdb links prom_stdout_is_framebuffer into its own 2861 * module, we add "device-type=display" here for /os-io node, so that 2862 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 2863 * is registered into OBP. 2864 */ 2865 static char *create_node = 2866 "\" /\" find-device " 2867 "new-device " 2868 "\" os-io\" device-name " 2869 "\" display\" device-type " 2870 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2871 " 2>r swap 2 2r> ['] $callback catch if " 2872 " 2drop 3drop 0 " 2873 " then " 2874 "; " 2875 ": read ( adr,len -- #read ) " 2876 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2877 " ( retN ... ret1 N ) " 2878 " ?dup if " 2879 " swap >r 1- 0 ?do drop loop r> " 2880 " else " 2881 " -2 " 2882 " then " 2883 "; " 2884 ": write ( adr,len -- #written ) " 2885 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2886 " ( retN ... ret1 N ) " 2887 " ?dup if " 2888 " swap >r 1- 0 ?do drop loop r> " 2889 " else " 2890 " 0 " 2891 " then " 2892 "; " 2893 ": poll-tty ( -- ) ; " 2894 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2895 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2896 ": cb-give/take ( $method -- ) " 2897 " 0 -rot ['] $callback catch ?dup if " 2898 " >r 2drop 2drop r> throw " 2899 " else " 2900 " 0 ?do drop loop " 2901 " then " 2902 "; " 2903 ": give ( -- ) \" exit-input\" cb-give/take ; " 2904 ": take ( -- ) \" enter-input\" cb-give/take ; " 2905 ": open ( -- ok? ) true ; " 2906 ": close ( -- ) ; " 2907 "finish-device " 2908 "device-end "; 2909 2910 /* 2911 * Create the OBP input/output node (FCode serial driver). 2912 * It is needed for both USB console keyboard and for 2913 * the kernel terminal emulator. It is too early to check for a 2914 * kernel console compatible framebuffer now, so we create this 2915 * so that we're ready if we need to enable kernel terminal emulation. 2916 * 2917 * When the USB software takes over the input device at the time 2918 * consconfig runs, OBP's stdin is redirected to this node. 2919 * Whenever the FORTH user interface is used after this switch, 2920 * the node will call back into the kernel for console input. 2921 * If a serial device such as ttya or a UART with a Type 5 keyboard 2922 * attached is used, OBP takes over the serial device when the system 2923 * goes to the debugger after the system is booted. This sharing 2924 * of the relatively simple serial device is difficult but possible. 2925 * Sharing the USB host controller is impossible due its complexity. 2926 * 2927 * Similarly to USB keyboard input redirection, after consconfig_dacf 2928 * configures a kernel console framebuffer as the standard output 2929 * device, OBP's stdout is switched to to vector through the 2930 * /os-io node into the kernel terminal emulator. 2931 */ 2932 static void 2933 startup_create_io_node(void) 2934 { 2935 prom_interpret(create_node, 0, 0, 0, 0, 0); 2936 } 2937 2938 2939 static void 2940 do_prom_version_check(void) 2941 { 2942 int i; 2943 pnode_t node; 2944 char buf[64]; 2945 static char drev[] = "Down-rev firmware detected%s\n" 2946 "\tPlease upgrade to the following minimum version:\n" 2947 "\t\t%s\n"; 2948 2949 i = prom_version_check(buf, sizeof (buf), &node); 2950 2951 if (i == PROM_VER64_OK) 2952 return; 2953 2954 if (i == PROM_VER64_UPGRADE) { 2955 cmn_err(CE_WARN, drev, "", buf); 2956 2957 #ifdef DEBUG 2958 prom_enter_mon(); /* Type 'go' to continue */ 2959 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2960 return; 2961 #else 2962 halt(0); 2963 #endif 2964 } 2965 2966 /* 2967 * The other possibility is that this is a server running 2968 * good firmware, but down-rev firmware was detected on at 2969 * least one other cpu board. We just complain if we see 2970 * that. 2971 */ 2972 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2973 } 2974 2975 static void 2976 kpm_init() 2977 { 2978 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 2979 kpm_pgsz = 1ull << kpm_pgshft; 2980 kpm_pgoff = kpm_pgsz - 1; 2981 kpmp2pshft = kpm_pgshft - PAGESHIFT; 2982 kpmpnpgs = 1 << kpmp2pshft; 2983 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 2984 } 2985 2986 void 2987 kpm_npages_setup(int memblocks) 2988 { 2989 /* 2990 * npages can be scattered in a maximum of 'memblocks' 2991 */ 2992 kpm_npages = ptokpmpr(npages) + memblocks; 2993 } 2994 2995 /* 2996 * Must be defined in platform dependent code. 2997 */ 2998 extern caddr_t modtext; 2999 extern size_t modtext_sz; 3000 extern caddr_t moddata; 3001 3002 #define HEAPTEXT_ARENA(addr) \ 3003 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3004 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3005 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3006 3007 #define HEAPTEXT_OVERSIZED(addr) \ 3008 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3009 3010 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3011 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3012 kmutex_t texthole_lock; 3013 3014 char kern_bootargs[OBP_MAXPATHLEN]; 3015 3016 void 3017 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3018 { 3019 uintptr_t addr, limit; 3020 3021 addr = HEAPTEXT_BASE; 3022 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3023 3024 /* 3025 * Before we initialize the text_arena, we want to punch holes in the 3026 * underlying heaptext_arena. This guarantees that for any text 3027 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3028 */ 3029 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3030 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3031 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3032 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3033 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3034 } 3035 3036 /* 3037 * Allocate one page at the oversize to break up the text region 3038 * from the oversized region. 3039 */ 3040 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3041 (void *)limit, (void *)(limit + PAGESIZE), 3042 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3043 3044 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3045 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3046 heaptext_arena, 0, VM_SLEEP); 3047 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3048 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3049 } 3050 3051 caddr_t 3052 kobj_text_alloc(vmem_t *arena, size_t size) 3053 { 3054 caddr_t rval, better; 3055 3056 /* 3057 * First, try a sleeping allocation. 3058 */ 3059 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3060 3061 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3062 return (rval); 3063 3064 /* 3065 * We didn't get the area that we wanted. We're going to try to do an 3066 * allocation with explicit constraints. 3067 */ 3068 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3069 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3070 VM_NOSLEEP | VM_BESTFIT); 3071 3072 if (better != NULL) { 3073 /* 3074 * That worked. Free our first attempt and return. 3075 */ 3076 vmem_free(arena, rval, size); 3077 return (better); 3078 } 3079 3080 /* 3081 * That didn't work; we'll have to return our first attempt. 3082 */ 3083 return (rval); 3084 } 3085 3086 caddr_t 3087 kobj_texthole_alloc(caddr_t addr, size_t size) 3088 { 3089 int arena = HEAPTEXT_ARENA(addr); 3090 char c[30]; 3091 uintptr_t base; 3092 3093 if (HEAPTEXT_OVERSIZED(addr)) { 3094 /* 3095 * If this is an oversized allocation, there is no text hole 3096 * available for it; return NULL. 3097 */ 3098 return (NULL); 3099 } 3100 3101 mutex_enter(&texthole_lock); 3102 3103 if (texthole_arena[arena] == NULL) { 3104 ASSERT(texthole_source[arena] == NULL); 3105 3106 if (arena == 0) { 3107 texthole_source[0] = vmem_create("module_text_holesrc", 3108 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3109 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3110 0, VM_SLEEP); 3111 } else { 3112 base = HEAPTEXT_BASE + 3113 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3114 3115 (void) snprintf(c, sizeof (c), 3116 "heaptext_holesrc_%d", arena); 3117 3118 texthole_source[arena] = vmem_create(c, (void *)base, 3119 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3120 0, VM_SLEEP); 3121 } 3122 3123 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3124 3125 texthole_arena[arena] = vmem_create(c, NULL, 0, 3126 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3127 texthole_source[arena], 0, VM_SLEEP); 3128 } 3129 3130 mutex_exit(&texthole_lock); 3131 3132 ASSERT(texthole_arena[arena] != NULL); 3133 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3134 return (vmem_alloc(texthole_arena[arena], size, 3135 VM_BESTFIT | VM_NOSLEEP)); 3136 } 3137 3138 void 3139 kobj_texthole_free(caddr_t addr, size_t size) 3140 { 3141 int arena = HEAPTEXT_ARENA(addr); 3142 3143 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3144 ASSERT(texthole_arena[arena] != NULL); 3145 vmem_free(texthole_arena[arena], addr, size); 3146 } 3147