1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 68 extern void setup_trap_table(void); 69 extern void cpu_intrq_setup(struct cpu *); 70 extern void cpu_intrq_register(struct cpu *); 71 extern void contig_mem_init(void); 72 extern void mach_dump_buffer_init(void); 73 extern void mach_descrip_init(void); 74 extern void mach_descrip_startup_fini(void); 75 extern void mach_memscrub(void); 76 extern void mach_fpras(void); 77 extern void mach_cpu_halt_idle(void); 78 extern void mach_hw_copy_limit(void); 79 extern void load_mach_drivers(void); 80 extern void load_tod_module(void); 81 #pragma weak load_tod_module 82 83 extern int ndata_alloc_mmfsa(struct memlist *ndata); 84 #pragma weak ndata_alloc_mmfsa 85 86 extern void cif_init(void); 87 #pragma weak cif_init 88 89 extern void parse_idprom(void); 90 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91 extern void mem_config_init(void); 92 extern void memseg_remap_init(void); 93 94 /* 95 * External Data: 96 */ 97 extern int vac_size; /* cache size in bytes */ 98 extern uint_t vac_mask; /* VAC alignment consistency mask */ 99 extern uint_t vac_colors; 100 101 /* 102 * Global Data Definitions: 103 */ 104 105 /* 106 * XXX - Don't port this to new architectures 107 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 108 * 'romp' has no use with a prom with an IEEE 1275 client interface. 109 * The driver doesn't use the value, but it depends on the symbol. 110 */ 111 void *romp; /* veritas driver won't load without romp 4154976 */ 112 /* 113 * Declare these as initialized data so we can patch them. 114 */ 115 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 116 pgcnt_t segkpsize = 117 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 118 uint_t segmap_percent = 12; /* Size of segmap segment */ 119 120 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 121 int vac_copyback = 1; 122 char *cache_mode = NULL; 123 int use_mix = 1; 124 int prom_debug = 0; 125 126 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 127 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 128 uint_t tba_taken_over = 0; 129 130 caddr_t s_text; /* start of kernel text segment */ 131 caddr_t e_text; /* end of kernel text segment */ 132 caddr_t s_data; /* start of kernel data segment */ 133 caddr_t e_data; /* end of kernel data segment */ 134 135 caddr_t modtext; /* beginning of module text */ 136 size_t modtext_sz; /* size of module text */ 137 caddr_t moddata; /* beginning of module data reserve */ 138 caddr_t e_moddata; /* end of module data reserve */ 139 140 /* 141 * End of first block of contiguous kernel in 32-bit virtual address space 142 */ 143 caddr_t econtig32; /* end of first blk of contiguous kernel */ 144 145 caddr_t ncbase; /* beginning of non-cached segment */ 146 caddr_t ncend; /* end of non-cached segment */ 147 caddr_t sdata; /* beginning of data segment */ 148 149 caddr_t extra_etva; /* beginning of unused nucleus text */ 150 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 151 152 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 153 caddr_t nalloc_base; /* beginning of nucleus allocation */ 154 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 155 caddr_t valloc_base; /* beginning of kvalloc segment */ 156 157 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 158 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 159 160 uintptr_t shm_alignment = 0; /* VAC address consistency modulus */ 161 struct memlist *phys_install; /* Total installed physical memory */ 162 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 163 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 164 struct memlist ndata; /* memlist of nucleus allocatable memory */ 165 int memexp_flag; /* memory expansion card flag */ 166 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 167 pgcnt_t obp_pages; /* Physical pages used by OBP */ 168 169 /* 170 * VM data structures 171 */ 172 long page_hashsz; /* Size of page hash table (power of two) */ 173 struct page *pp_base; /* Base of system page struct array */ 174 size_t pp_sz; /* Size in bytes of page struct array */ 175 struct page **page_hash; /* Page hash table */ 176 struct seg ktextseg; /* Segment used for kernel executable image */ 177 struct seg kvalloc; /* Segment used for "valloc" mapping */ 178 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 179 struct seg ktexthole; /* Segment used for nucleus text hole */ 180 struct seg kmapseg; /* Segment used for generic kernel mappings */ 181 struct seg kpmseg; /* Segment used for physical mapping */ 182 struct seg kdebugseg; /* Segment used for the kernel debugger */ 183 184 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 185 size_t kpm_pp_sz; /* Size of system kpm_page array */ 186 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 187 188 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 189 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 190 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 191 192 /* 193 * debugger pages (if allocated) 194 */ 195 struct vnode kdebugvp; 196 197 /* 198 * Segment for relocated kernel structures in 64-bit large RAM kernels 199 */ 200 struct seg kmem64; 201 202 struct memseg *memseg_base; 203 size_t memseg_sz; /* Used to translate a va to page */ 204 struct vnode unused_pages_vp; 205 206 /* 207 * VM data structures allocated early during boot. 208 */ 209 size_t pagehash_sz; 210 uint64_t memlist_sz; 211 212 char tbr_wr_addr_inited = 0; 213 214 215 /* 216 * Static Routines: 217 */ 218 static void memlist_add(uint64_t, uint64_t, struct memlist **, 219 struct memlist **); 220 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 221 pgcnt_t); 222 static void kvm_init(void); 223 224 static void startup_init(void); 225 static void startup_memlist(void); 226 static void startup_modules(void); 227 static void startup_bop_gone(void); 228 static void startup_vm(void); 229 static void startup_end(void); 230 static void setup_cage_params(void); 231 static void startup_create_io_node(void); 232 233 static pgcnt_t npages; 234 static struct memlist *memlist; 235 void *memlist_end; 236 237 static pgcnt_t bop_alloc_pages; 238 static caddr_t hblk_base; 239 uint_t hblk_alloc_dynamic = 0; 240 uint_t hblk1_min = H1MIN; 241 uint_t hblk8_min; 242 243 244 /* 245 * Hooks for unsupported platforms and down-rev firmware 246 */ 247 int iam_positron(void); 248 #pragma weak iam_positron 249 static void do_prom_version_check(void); 250 static void kpm_init(void); 251 static void kpm_npages_setup(int); 252 static void kpm_memseg_init(void); 253 254 /* 255 * After receiving a thermal interrupt, this is the number of seconds 256 * to delay before shutting off the system, assuming 257 * shutdown fails. Use /etc/system to change the delay if this isn't 258 * large enough. 259 */ 260 int thermal_powerdown_delay = 1200; 261 262 /* 263 * Used to hold off page relocations into the cage until OBP has completed 264 * its boot-time handoff of its resources to the kernel. 265 */ 266 int page_relocate_ready = 0; 267 268 /* 269 * Enable some debugging messages concerning memory usage... 270 */ 271 #ifdef DEBUGGING_MEM 272 static int debugging_mem; 273 static void 274 printmemlist(char *title, struct memlist *list) 275 { 276 if (!debugging_mem) 277 return; 278 279 printf("%s\n", title); 280 281 while (list) { 282 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 283 (uint32_t)(list->address >> 32), (uint32_t)list->address, 284 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 285 list = list->next; 286 } 287 } 288 289 void 290 printmemseg(struct memseg *memseg) 291 { 292 if (!debugging_mem) 293 return; 294 295 printf("memseg\n"); 296 297 while (memseg) { 298 prom_printf("\tpage = 0x%p, epage = 0x%p, " 299 "pfn = 0x%x, epfn = 0x%x\n", 300 memseg->pages, memseg->epages, 301 memseg->pages_base, memseg->pages_end); 302 memseg = memseg->next; 303 } 304 } 305 306 #define debug_pause(str) halt((str)) 307 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 308 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 309 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 310 #define MPRINTF3(str, a, b, c) \ 311 if (debugging_mem) prom_printf((str), (a), (b), (c)) 312 #else /* DEBUGGING_MEM */ 313 #define MPRINTF(str) 314 #define MPRINTF1(str, a) 315 #define MPRINTF2(str, a, b) 316 #define MPRINTF3(str, a, b, c) 317 #endif /* DEBUGGING_MEM */ 318 319 /* Simple message to indicate that the bootops pointer has been zeroed */ 320 #ifdef DEBUG 321 static int bootops_gone_on = 0; 322 #define BOOTOPS_GONE() \ 323 if (bootops_gone_on) \ 324 prom_printf("The bootops vec is zeroed now!\n"); 325 #else 326 #define BOOTOPS_GONE() 327 #endif /* DEBUG */ 328 329 /* 330 * Monitor pages may not be where this says they are. 331 * and the debugger may not be there either. 332 * 333 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 334 * 335 * Physical memory layout 336 * (not necessarily contiguous) 337 * (THIS IS SOMEWHAT WRONG) 338 * /-----------------------\ 339 * | monitor pages | 340 * availmem -|-----------------------| 341 * | | 342 * | page pool | 343 * | | 344 * |-----------------------| 345 * | configured tables | 346 * | buffers | 347 * firstaddr -|-----------------------| 348 * | hat data structures | 349 * |-----------------------| 350 * | kernel data, bss | 351 * |-----------------------| 352 * | interrupt stack | 353 * |-----------------------| 354 * | kernel text (RO) | 355 * |-----------------------| 356 * | trap table (4k) | 357 * |-----------------------| 358 * page 1 | panicbuf | 359 * |-----------------------| 360 * page 0 | reclaimed | 361 * |_______________________| 362 * 363 * 364 * 365 * Kernel's Virtual Memory Layout. 366 * /-----------------------\ 367 * 0xFFFFFFFF.FFFFFFFF -| |- 368 * | OBP's virtual page | 369 * | tables | 370 * 0xFFFFFFFC.00000000 -|-----------------------|- 371 * : : 372 * : : 373 * 0xFFFFFE00.00000000 -|-----------------------|- 374 * | | Ultrasparc I/II support 375 * | segkpm segment | up to 2TB of physical 376 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 377 * | | 378 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 379 * : : 380 * : : 381 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 382 * | | ^ 383 * | UltraSPARC I/II call | | 384 * | bug requires an extra | | 385 * | 4 GB of space between | | 386 * | hole and used RAM | | 387 * | | | 388 * 0xFFFFF800.00000000 -|-----------------------|- | 389 * | | | 390 * | Virtual Address Hole | UltraSPARC 391 * | on UltraSPARC I/II | I/II * ONLY * 392 * | | | 393 * 0x00000800.00000000 -|-----------------------|- | 394 * | | | 395 * | UltraSPARC I/II call | | 396 * | bug requires an extra | | 397 * | 4 GB of space between | | 398 * | hole and used RAM | | 399 * | | v 400 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 401 * : : ^ 402 * : : | 403 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 404 * | | | 405 * | 64-bit kernel ONLY | | 406 * | | | 407 * | kmem64 segment | | 408 * | | | 409 * | (Relocated extra HME | Approximately 410 * | block allocations, | 1 TB of virtual 411 * | memnode freelists, | address space 412 * | HME hash buckets, | | 413 * | mml_table, kpmp_table,| | 414 * | page_t array and | | 415 * | hashblock pool to | | 416 * | avoid hard-coded | | 417 * | 32-bit vaddr | | 418 * | limitations) | | 419 * | | v 420 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 421 * | | 422 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 423 * | | 424 * 0x00000300.00000000 -|-----------------------|- SYSBASE 425 * : : 426 * : : 427 * -|-----------------------|- 428 * | | 429 * | segmap segment | SEGMAPSIZE (1/8th physmem, 430 * | | 256G MAX) 431 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 432 * : : 433 * : : 434 * -|-----------------------|- 435 * | | 436 * | segkp | SEGKPSIZE (2GB) 437 * | | 438 * | | 439 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 440 * | | 441 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 442 * | | (SEGKPBASE - 0x400000) 443 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 444 * | | (MEMSCRUBBASE - NCARGS) 445 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 446 * | | (ARGSBASE - PPMAPSIZE) 447 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 448 * | | 449 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 450 * | | 451 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 452 * : : 453 * : : 454 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 455 * | | 456 * | OBP | 457 * | | 458 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 459 * | kmdb | 460 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 461 * : : 462 * : : 463 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 464 * | | 465 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 466 * | | ~64MB) 467 * 0x00000000.78002000 -|-----------------------| 468 * | panicbuf | 469 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 470 * : : 471 * : : 472 * | | 473 * |-----------------------|- econtig32 474 * | vm structures | 475 * 0x00000000.01C00000 |-----------------------|- nalloc_end 476 * | TSBs | 477 * |-----------------------|- end/nalloc_base 478 * | kernel data & bss | 479 * 0x00000000.01800000 -|-----------------------| 480 * : nucleus text hole : 481 * 0x00000000.01400000 -|-----------------------| 482 * : : 483 * |-----------------------| 484 * | module text | 485 * |-----------------------|- e_text/modtext 486 * | kernel text | 487 * |-----------------------| 488 * | trap table (48k) | 489 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 490 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 491 * |-----------------------| 492 * | | 493 * | invalid | 494 * | | 495 * 0x00000000.00000000 _|_______________________| 496 * 497 * 498 * 499 * 32-bit User Virtual Memory Layout. 500 * /-----------------------\ 501 * | | 502 * | invalid | 503 * | | 504 * 0xFFC00000 -|-----------------------|- USERLIMIT 505 * | user stack | 506 * : : 507 * : : 508 * : : 509 * | user data | 510 * -|-----------------------|- 511 * | user text | 512 * 0x00002000 -|-----------------------|- 513 * | invalid | 514 * 0x00000000 _|_______________________| 515 * 516 * 517 * 518 * 64-bit User Virtual Memory Layout. 519 * /-----------------------\ 520 * | | 521 * | invalid | 522 * | | 523 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 524 * | user stack | 525 * : : 526 * : : 527 * : : 528 * | user data | 529 * -|-----------------------|- 530 * | user text | 531 * 0x00000000.00100000 -|-----------------------|- 532 * | invalid | 533 * 0x00000000.00000000 _|_______________________| 534 */ 535 536 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 537 extern uint64_t ecache_flush_address(void); 538 539 #pragma weak load_platform_modules 540 #pragma weak plat_startup_memlist 541 #pragma weak ecache_init_scrub_flush_area 542 #pragma weak ecache_flush_address 543 544 545 /* 546 * By default the DR Cage is enabled for maximum OS 547 * MPSS performance. Users needing to disable the cage mechanism 548 * can set this variable to zero via /etc/system. 549 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 550 * will result in loss of DR functionality. 551 * Platforms wishing to disable kernel Cage by default 552 * should do so in their set_platform_defaults() routine. 553 */ 554 int kernel_cage_enable = 1; 555 556 static void 557 setup_cage_params(void) 558 { 559 void (*func)(void); 560 561 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 562 if (func != NULL) { 563 (*func)(); 564 return; 565 } 566 567 if (kernel_cage_enable == 0) { 568 return; 569 } 570 kcage_range_lock(); 571 if (kcage_range_init(phys_avail, 1) == 0) { 572 kcage_init(total_pages / 256); 573 } 574 kcage_range_unlock(); 575 576 if (kcage_on) { 577 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 578 } else { 579 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 580 } 581 582 } 583 584 /* 585 * Machine-dependent startup code 586 */ 587 void 588 startup(void) 589 { 590 startup_init(); 591 if (&startup_platform) 592 startup_platform(); 593 startup_memlist(); 594 startup_modules(); 595 setup_cage_params(); 596 startup_bop_gone(); 597 startup_vm(); 598 startup_end(); 599 } 600 601 struct regs sync_reg_buf; 602 uint64_t sync_tt; 603 604 void 605 sync_handler(void) 606 { 607 struct trap_info ti; 608 int i; 609 610 /* 611 * Prevent trying to talk to the other CPUs since they are 612 * sitting in the prom and won't reply. 613 */ 614 for (i = 0; i < NCPU; i++) { 615 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 616 cpu[i]->cpu_flags &= ~CPU_READY; 617 cpu[i]->cpu_flags |= CPU_QUIESCED; 618 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 619 } 620 } 621 622 /* 623 * We've managed to get here without going through the 624 * normal panic code path. Try and save some useful 625 * information. 626 */ 627 if (!panicstr && (curthread->t_panic_trap == NULL)) { 628 ti.trap_type = sync_tt; 629 ti.trap_regs = &sync_reg_buf; 630 ti.trap_addr = NULL; 631 ti.trap_mmu_fsr = 0x0; 632 633 curthread->t_panic_trap = &ti; 634 } 635 636 /* 637 * If we're re-entering the panic path, update the signature 638 * block so that the SC knows we're in the second part of panic. 639 */ 640 if (panicstr) 641 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 642 643 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 644 panic("sync initiated"); 645 } 646 647 648 static void 649 startup_init(void) 650 { 651 /* 652 * We want to save the registers while we're still in OBP 653 * so that we know they haven't been fiddled with since. 654 * (In principle, OBP can't change them just because it 655 * makes a callback, but we'd rather not depend on that 656 * behavior.) 657 */ 658 char sync_str[] = 659 "warning @ warning off : sync " 660 "%%tl-c %%tstate h# %p x! " 661 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 662 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 663 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 664 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 665 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 666 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 667 "%%y h# %p l! %%tl-c %%tt h# %p x! " 668 "sync ; warning !"; 669 670 /* 671 * 20 == num of %p substrings 672 * 16 == max num of chars %p will expand to. 673 */ 674 char bp[sizeof (sync_str) + 16 * 20]; 675 676 (void) check_boot_version(BOP_GETVERSION(bootops)); 677 678 /* 679 * Initialize ptl1 stack for the 1st CPU. 680 */ 681 ptl1_init_cpu(&cpu0); 682 683 /* 684 * Initialize the address map for cache consistent mappings 685 * to random pages; must be done after vac_size is set. 686 */ 687 ppmapinit(); 688 689 /* 690 * Initialize the PROM callback handler. 691 */ 692 init_vx_handler(); 693 694 /* 695 * have prom call sync_callback() to handle the sync and 696 * save some useful information which will be stored in the 697 * core file later. 698 */ 699 (void) sprintf((char *)bp, sync_str, 700 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 701 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 702 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 703 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 704 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 705 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 706 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 707 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 708 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 709 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 710 prom_interpret(bp, 0, 0, 0, 0, 0); 711 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 712 } 713 714 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 715 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 716 717 #define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector)) 718 719 /* 720 * As OBP takes up some RAM when the system boots, pages will already be "lost" 721 * to the system and reflected in npages by the time we see it. 722 * 723 * We only want to allocate kernel structures in the 64-bit virtual address 724 * space on systems with enough RAM to make the overhead of keeping track of 725 * an extra kernel memory segment worthwhile. 726 * 727 * Since OBP has already performed its memory allocations by this point, if we 728 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 729 * memory in the 64-bit virtual address space; otherwise keep allocations 730 * contiguous with we've mapped so far in the 32-bit virtual address space. 731 */ 732 #define MINMOVE_RAM_MB ((size_t)1900) 733 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 734 735 pgcnt_t tune_npages = (pgcnt_t) 736 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 737 738 static void 739 startup_memlist(void) 740 { 741 size_t alloc_sz; 742 size_t ctrs_sz; 743 caddr_t alloc_base; 744 caddr_t ctrs_base, ctrs_end; 745 caddr_t memspace; 746 caddr_t va; 747 int memblocks = 0; 748 struct memlist *cur; 749 size_t syslimit = (size_t)SYSLIMIT; 750 size_t sysbase = (size_t)SYSBASE; 751 int alloc_alignsize = MMU_PAGESIZE; 752 extern void page_coloring_init(void); 753 754 /* 755 * Initialize enough of the system to allow kmem_alloc to work by 756 * calling boot to allocate its memory until the time that 757 * kvm_init is completed. The page structs are allocated after 758 * rounding up end to the nearest page boundary; the memsegs are 759 * initialized and the space they use comes from the kernel heap. 760 * With appropriate initialization, they can be reallocated later 761 * to a size appropriate for the machine's configuration. 762 * 763 * At this point, memory is allocated for things that will never 764 * need to be freed, this used to be "valloced". This allows a 765 * savings as the pages don't need page structures to describe 766 * them because them will not be managed by the vm system. 767 */ 768 769 /* 770 * We're loaded by boot with the following configuration (as 771 * specified in the sun4u/conf/Mapfile): 772 * 773 * text: 4 MB chunk aligned on a 4MB boundary 774 * data & bss: 4 MB chunk aligned on a 4MB boundary 775 * 776 * These two chunks will eventually be mapped by 2 locked 4MB 777 * ttes and will represent the nucleus of the kernel. This gives 778 * us some free space that is already allocated, some or all of 779 * which is made available to kernel module text. 780 * 781 * The free space in the data-bss chunk is used for nucleus 782 * allocatable data structures and we reserve it using the 783 * nalloc_base and nalloc_end variables. This space is currently 784 * being used for hat data structures required for tlb miss 785 * handling operations. We align nalloc_base to a l2 cache 786 * linesize because this is the line size the hardware uses to 787 * maintain cache coherency. 788 * 256K is carved out for module data. 789 */ 790 791 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 792 moddata = nalloc_base; 793 e_moddata = nalloc_base + MODDATA; 794 nalloc_base = e_moddata; 795 796 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 797 valloc_base = nalloc_base; 798 799 /* 800 * Calculate the start of the data segment. 801 */ 802 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 803 804 PRM_DEBUG(moddata); 805 PRM_DEBUG(nalloc_base); 806 PRM_DEBUG(nalloc_end); 807 PRM_DEBUG(sdata); 808 809 /* 810 * Remember any slop after e_text so we can give it to the modules. 811 */ 812 PRM_DEBUG(e_text); 813 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 814 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 815 panic("nucleus text overflow"); 816 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 817 modtext; 818 PRM_DEBUG(modtext); 819 PRM_DEBUG(modtext_sz); 820 821 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 822 &boot_physavail, &boot_physavail_len, 823 &boot_virtavail, &boot_virtavail_len); 824 /* 825 * Remember what the physically available highest page is 826 * so that dumpsys works properly, and find out how much 827 * memory is installed. 828 */ 829 installed_top_size_memlist_array(boot_physinstalled, 830 boot_physinstalled_len, &physmax, &physinstalled); 831 PRM_DEBUG(physinstalled); 832 PRM_DEBUG(physmax); 833 834 /* Fill out memory nodes config structure */ 835 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 836 837 /* 838 * Get the list of physically available memory to size 839 * the number of page structures needed. 840 */ 841 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 842 /* 843 * This first snap shot of npages can represent the pages used 844 * by OBP's text and data approximately. This is used in the 845 * the calculation of the kernel size 846 */ 847 obp_pages = physinstalled - npages; 848 849 850 /* 851 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 852 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 853 * 2MB) and any excess pages are put on physavail. The assumption is 854 * that small-memory systems will need more pages more than they'll 855 * need efficiently-mapped module texts. 856 */ 857 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 858 modtext_sz > MODTEXT_SM_CAP) { 859 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 860 modtext_sz = MODTEXT_SM_CAP; 861 } else 862 extra_etpg = 0; 863 PRM_DEBUG(extra_etpg); 864 PRM_DEBUG(modtext_sz); 865 extra_etva = modtext + modtext_sz; 866 PRM_DEBUG(extra_etva); 867 868 /* 869 * Account for any pages after e_text and e_data. 870 */ 871 npages += extra_etpg; 872 npages += mmu_btopr(nalloc_end - nalloc_base); 873 PRM_DEBUG(npages); 874 875 /* 876 * npages is the maximum of available physical memory possible. 877 * (ie. it will never be more than this) 878 */ 879 880 /* 881 * initialize the nucleus memory allocator. 882 */ 883 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 884 885 /* 886 * Allocate mmu fault status area from the nucleus data area. 887 */ 888 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 889 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 890 891 /* 892 * Allocate kernel TSBs from the nucleus data area. 893 */ 894 if (ndata_alloc_tsbs(&ndata, npages) != 0) 895 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 896 897 /* 898 * Allocate cpus structs from the nucleus data area. 899 */ 900 if (ndata_alloc_cpus(&ndata) != 0) 901 cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc"); 902 903 /* 904 * Allocate dmv dispatch table from the nucleus data area. 905 */ 906 if (ndata_alloc_dmv(&ndata) != 0) 907 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 908 909 910 page_coloring_init(); 911 912 /* 913 * Allocate page_freelists bin headers for memnode 0 from the 914 * nucleus data area. 915 */ 916 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 917 cmn_err(CE_PANIC, 918 "no more nucleus memory after page free lists alloc"); 919 920 if (kpm_enable) { 921 kpm_init(); 922 /* 923 * kpm page space -- Update kpm_npages and make the 924 * same assumption about fragmenting as it is done 925 * for memseg_sz. 926 */ 927 kpm_npages_setup(memblocks + 4); 928 } 929 930 /* 931 * Allocate hat related structs from the nucleus data area. 932 */ 933 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 934 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 935 936 /* 937 * We want to do the BOP_ALLOCs before the real allocation of page 938 * structs in order to not have to allocate page structs for this 939 * memory. We need to calculate a virtual address because we want 940 * the page structs to come before other allocations in virtual address 941 * space. This is so some (if not all) of page structs can actually 942 * live in the nucleus. 943 */ 944 945 /* 946 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 947 * 948 * There are comments all over the SFMMU code warning of dire 949 * consequences if the TSBs are moved out of 32-bit space. This 950 * is largely because the asm code uses "sethi %hi(addr)"-type 951 * instructions which will not provide the expected result if the 952 * address is a 64-bit one. 953 * 954 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 955 */ 956 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 957 alloc_base = sfmmu_ktsb_alloc(alloc_base); 958 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 959 PRM_DEBUG(alloc_base); 960 961 /* 962 * Allocate IOMMU TSB array. We do this here so that the physical 963 * memory gets deducted from the PROM's physical memory list. 964 */ 965 alloc_base = iommu_tsb_init(alloc_base); 966 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 967 ecache_alignsize); 968 PRM_DEBUG(alloc_base); 969 970 /* 971 * Platforms like Starcat and OPL need special structures assigned in 972 * 32-bit virtual address space because their probing routines execute 973 * FCode, and FCode can't handle 64-bit virtual addresses... 974 */ 975 if (&plat_startup_memlist) { 976 alloc_base = plat_startup_memlist(alloc_base); 977 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 978 ecache_alignsize); 979 PRM_DEBUG(alloc_base); 980 } 981 982 /* 983 * If we have enough memory, use 4M pages for alignment because it 984 * greatly reduces the number of TLB misses we take albeit at the cost 985 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 986 * allocation.) Still, the speedup on large memory systems (e.g. > 64 987 * GB) is quite noticeable, so it is worth the effort to do if we can. 988 * 989 * Note, however, that this speedup will only occur if the boot PROM 990 * uses the largest possible MMU page size possible to map memory 991 * requests that are properly aligned and sized (for example, a request 992 * for a multiple of 4MB of memory aligned to a 4MB boundary will 993 * result in a mapping using a 4MB MMU page.) 994 * 995 * Even then, the large page mappings will only speed things up until 996 * the startup process proceeds a bit further, as when 997 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 998 * kernel it remaps everything but the TSBs using 8K pages anyway... 999 * 1000 * At some point in the future, sfmmu_map_prom_mappings() will be 1001 * rewritten to copy memory mappings to the kernel using the same MMU 1002 * page sizes the PROM used. When that occurs, if the PROM did use 1003 * large MMU pages to map memory, the alignment/sizing work we're 1004 * doing now should give us a nice extra performance boost, albeit at 1005 * the cost of greater RAM usage... 1006 */ 1007 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1008 MMU_PAGESIZE); 1009 1010 PRM_DEBUG(tune_npages); 1011 PRM_DEBUG(alloc_alignsize); 1012 1013 /* 1014 * Save off where the contiguous allocations to date have ended 1015 * in econtig32. 1016 */ 1017 econtig32 = alloc_base; 1018 PRM_DEBUG(econtig32); 1019 1020 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1021 cmn_err(CE_PANIC, "econtig32 too big"); 1022 1023 /* 1024 * To avoid memory allocation collisions in the 32-bit virtual address 1025 * space, make allocations from this point forward in 64-bit virtual 1026 * address space starting at syslimit and working up. Also use the 1027 * alignment specified by alloc_alignsize, as we may be able to save 1028 * ourselves TLB misses by using larger page sizes if they're 1029 * available. 1030 * 1031 * All this is needed because on large memory systems, the default 1032 * Solaris allocations will collide with SYSBASE32, which is hard 1033 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1034 * kernels, move the allocations to a location in the 64-bit virtual 1035 * address space space, allowing those structures to grow without 1036 * worry. 1037 * 1038 * On current CPUs we'll run out of physical memory address bits before 1039 * we need to worry about the allocations running into anything else in 1040 * VM or the virtual address holes on US-I and II, as there's currently 1041 * about 1 TB of addressable space before the US-I/II VA hole. 1042 */ 1043 kmem64_base = (caddr_t)syslimit; 1044 PRM_DEBUG(kmem64_base); 1045 1046 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1047 1048 /* 1049 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1050 * them here. 1051 */ 1052 if (khme_hash == NULL || uhme_hash == NULL) { 1053 /* 1054 * alloc_hme_buckets() will align alloc_base properly before 1055 * assigning the hash buckets, so we don't need to do it 1056 * before the call... 1057 */ 1058 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1059 1060 PRM_DEBUG(alloc_base); 1061 PRM_DEBUG(khme_hash); 1062 PRM_DEBUG(uhme_hash); 1063 } 1064 1065 /* 1066 * Allocate the remaining page freelists. NUMA systems can 1067 * have lots of page freelists, one per node, which quickly 1068 * outgrow the amount of nucleus memory available. 1069 */ 1070 if (max_mem_nodes > 1) { 1071 int mnode; 1072 caddr_t alloc_start = alloc_base; 1073 1074 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1075 alloc_base = alloc_page_freelists(mnode, alloc_base, 1076 ecache_alignsize); 1077 } 1078 1079 if (alloc_base > alloc_start) { 1080 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1081 alloc_alignsize); 1082 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1083 alloc_base - alloc_start, 1084 alloc_alignsize) != alloc_start) 1085 cmn_err(CE_PANIC, 1086 "Unable to alloc page freelists\n"); 1087 } 1088 1089 PRM_DEBUG(alloc_base); 1090 } 1091 1092 if (!mml_table) { 1093 size_t mmltable_sz; 1094 1095 /* 1096 * We need to allocate the mml_table here because there 1097 * was not enough space within the nucleus. 1098 */ 1099 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1100 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1101 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1102 alloc_alignsize); 1103 1104 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1105 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1106 panic("mml_table alloc failure"); 1107 1108 alloc_base += alloc_sz; 1109 PRM_DEBUG(mml_table); 1110 PRM_DEBUG(alloc_base); 1111 } 1112 1113 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1114 size_t kpmptable_sz; 1115 caddr_t table; 1116 1117 /* 1118 * We need to allocate either kpmp_table or kpmp_stable here 1119 * because there was not enough space within the nucleus. 1120 */ 1121 kpmptable_sz = (kpm_smallpages == 0) ? 1122 sizeof (kpm_hlk_t) * kpmp_table_sz : 1123 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1124 1125 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1126 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1127 alloc_alignsize); 1128 1129 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1130 alloc_alignsize); 1131 1132 if (table != alloc_base) 1133 panic("kpmp_table or kpmp_stable alloc failure"); 1134 1135 if (kpm_smallpages == 0) { 1136 kpmp_table = (kpm_hlk_t *)table; 1137 PRM_DEBUG(kpmp_table); 1138 } else { 1139 kpmp_stable = (kpm_shlk_t *)table; 1140 PRM_DEBUG(kpmp_stable); 1141 } 1142 1143 alloc_base += alloc_sz; 1144 PRM_DEBUG(alloc_base); 1145 } 1146 1147 if (&ecache_init_scrub_flush_area) { 1148 /* 1149 * Pass alloc_base directly, as the routine itself is 1150 * responsible for any special alignment requirements... 1151 */ 1152 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1153 PRM_DEBUG(alloc_base); 1154 } 1155 1156 /* 1157 * Take the most current snapshot we can by calling mem-update. 1158 */ 1159 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1160 &boot_physavail, &boot_physavail_len, 1161 &boot_virtavail, &boot_virtavail_len); 1162 1163 /* 1164 * Reset npages and memblocks based on boot_physavail list. 1165 */ 1166 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1167 PRM_DEBUG(npages); 1168 1169 /* 1170 * Account for extra memory after e_text. 1171 */ 1172 npages += extra_etpg; 1173 1174 /* 1175 * Calculate the largest free memory chunk in the nucleus data area. 1176 * We need to figure out if page structs can fit in there or not. 1177 * We also make sure enough page structs get created for any physical 1178 * memory we might be returning to the system. 1179 */ 1180 ndata_remain_sz = ndata_maxsize(&ndata); 1181 PRM_DEBUG(ndata_remain_sz); 1182 1183 pp_sz = sizeof (struct page) * npages; 1184 1185 /* 1186 * Here's a nice bit of code based on somewhat recursive logic: 1187 * 1188 * If the page array would fit within the nucleus, we want to 1189 * add npages to cover any extra memory we may be returning back 1190 * to the system. 1191 * 1192 * HOWEVER, the page array is sized by calculating the size of 1193 * (struct page * npages), as are the pagehash table, ctrs and 1194 * memseg_list, so the very act of performing the calculation below may 1195 * in fact make the array large enough that it no longer fits in the 1196 * nucleus, meaning there would now be a much larger area of the 1197 * nucleus free that should really be added to npages, which would 1198 * make the page array that much larger, and so on. 1199 * 1200 * This also ignores the memory possibly used in the nucleus for the 1201 * the page hash, ctrs and memseg list and the fact that whether they 1202 * fit there or not varies with the npages calculation below, but we 1203 * don't even factor them into the equation at this point; perhaps we 1204 * should or perhaps we should just take the approach that the few 1205 * extra pages we could add via this calculation REALLY aren't worth 1206 * the hassle... 1207 */ 1208 if (ndata_remain_sz > pp_sz) { 1209 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1210 1211 npages += mmu_btop(spare); 1212 1213 pp_sz = npages * sizeof (struct page); 1214 1215 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1216 } 1217 1218 /* 1219 * If physmem is patched to be non-zero, use it instead of 1220 * the monitor value unless physmem is larger than the total 1221 * amount of memory on hand. 1222 */ 1223 if (physmem == 0 || physmem > npages) 1224 physmem = npages; 1225 1226 /* 1227 * If pp_base is NULL that means the routines above have determined 1228 * the page array will not fit in the nucleus; we'll have to 1229 * BOP_ALLOC() ourselves some space for them. 1230 */ 1231 if (pp_base == NULL) { 1232 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1233 alloc_alignsize); 1234 1235 alloc_sz = roundup(pp_sz, alloc_alignsize); 1236 1237 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1238 alloc_base, alloc_sz, alloc_alignsize)) != 1239 (struct page *)alloc_base) 1240 panic("page alloc failure"); 1241 1242 alloc_base += alloc_sz; 1243 } 1244 1245 /* 1246 * The page structure hash table size is a power of 2 1247 * such that the average hash chain length is PAGE_HASHAVELEN. 1248 */ 1249 page_hashsz = npages / PAGE_HASHAVELEN; 1250 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1251 pagehash_sz = sizeof (struct page *) * page_hashsz; 1252 1253 /* 1254 * We want to TRY to fit the page structure hash table, 1255 * the page size free list counters, the memseg list and 1256 * and the kpm page space in the nucleus if possible. 1257 * 1258 * alloc_sz counts how much memory needs to be allocated by 1259 * BOP_ALLOC(). 1260 */ 1261 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1262 1263 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1264 1265 /* 1266 * Size up per page size free list counters. 1267 */ 1268 ctrs_sz = page_ctrs_sz(); 1269 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1270 1271 if (ctrs_base == NULL) 1272 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1273 1274 /* 1275 * The memseg list is for the chunks of physical memory that 1276 * will be managed by the vm system. The number calculated is 1277 * a guess as boot may fragment it more when memory allocations 1278 * are made before kphysm_init(). Currently, there are two 1279 * allocations before then, so we assume each causes fragmen- 1280 * tation, and add a couple more for good measure. 1281 */ 1282 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1283 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1284 1285 if (memseg_base == NULL) 1286 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1287 1288 1289 if (kpm_enable) { 1290 /* 1291 * kpm page space -- Update kpm_npages and make the 1292 * same assumption about fragmenting as it is done 1293 * for memseg_sz above. 1294 */ 1295 kpm_npages_setup(memblocks + 4); 1296 kpm_pp_sz = (kpm_smallpages == 0) ? 1297 kpm_npages * sizeof (kpm_page_t): 1298 kpm_npages * sizeof (kpm_spage_t); 1299 1300 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1301 ecache_alignsize); 1302 1303 if (kpm_pp_base == NULL) 1304 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1305 kpm_pp_sz; 1306 } 1307 1308 if (alloc_sz > 0) { 1309 uintptr_t bop_base; 1310 1311 /* 1312 * We need extra memory allocated through BOP_ALLOC. 1313 */ 1314 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1315 alloc_alignsize); 1316 1317 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1318 1319 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1320 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1321 panic("system page struct alloc failure"); 1322 1323 alloc_base += alloc_sz; 1324 1325 if (page_hash == NULL) { 1326 page_hash = (struct page **)bop_base; 1327 bop_base = roundup(bop_base + pagehash_sz, 1328 ecache_alignsize); 1329 } 1330 1331 if (ctrs_base == NULL) { 1332 ctrs_base = (caddr_t)bop_base; 1333 bop_base = roundup(bop_base + ctrs_sz, 1334 ecache_alignsize); 1335 } 1336 1337 if (memseg_base == NULL) { 1338 memseg_base = (struct memseg *)bop_base; 1339 bop_base = roundup(bop_base + memseg_sz, 1340 ecache_alignsize); 1341 } 1342 1343 if (kpm_enable && kpm_pp_base == NULL) { 1344 kpm_pp_base = (uintptr_t)bop_base; 1345 bop_base = roundup(bop_base + kpm_pp_sz, 1346 ecache_alignsize); 1347 } 1348 1349 ASSERT(bop_base <= (uintptr_t)alloc_base); 1350 } 1351 1352 /* 1353 * Initialize per page size free list counters. 1354 */ 1355 ctrs_end = page_ctrs_alloc(ctrs_base); 1356 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1357 1358 PRM_DEBUG(page_hash); 1359 PRM_DEBUG(memseg_base); 1360 PRM_DEBUG(kpm_pp_base); 1361 PRM_DEBUG(kpm_pp_sz); 1362 PRM_DEBUG(pp_base); 1363 PRM_DEBUG(pp_sz); 1364 PRM_DEBUG(alloc_base); 1365 1366 #ifdef TRAPTRACE 1367 /* 1368 * Allocate trap trace buffer last so as not to affect 1369 * the 4M alignments of the allocations above on V9 SPARCs... 1370 */ 1371 alloc_base = trap_trace_alloc(alloc_base); 1372 PRM_DEBUG(alloc_base); 1373 #endif /* TRAPTRACE */ 1374 1375 if (kmem64_base) { 1376 /* 1377 * Set the end of the kmem64 segment for V9 SPARCs, if 1378 * appropriate... 1379 */ 1380 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1381 alloc_alignsize); 1382 1383 PRM_DEBUG(kmem64_base); 1384 PRM_DEBUG(kmem64_end); 1385 } 1386 1387 /* 1388 * Allocate space for the interrupt vector table. 1389 */ 1390 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector, 1391 IVSIZE, MMU_PAGESIZE); 1392 if (memspace != (caddr_t)intr_vector) 1393 panic("interrupt table allocation failure"); 1394 1395 /* 1396 * The memory lists from boot are allocated from the heap arena 1397 * so that later they can be freed and/or reallocated. 1398 */ 1399 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1400 panic("could not retrieve property \"extent\""); 1401 1402 /* 1403 * Between now and when we finish copying in the memory lists, 1404 * allocations happen so the space gets fragmented and the 1405 * lists longer. Leave enough space for lists twice as long 1406 * as what boot says it has now; roundup to a pagesize. 1407 * Also add space for the final phys-avail copy in the fixup 1408 * routine. 1409 */ 1410 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1411 roundup(IVSIZE, MMU_PAGESIZE)); 1412 memlist_sz *= 4; 1413 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1414 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1415 if (memspace == NULL) 1416 halt("Boot allocation failed."); 1417 1418 memlist = (struct memlist *)memspace; 1419 memlist_end = (char *)memspace + memlist_sz; 1420 1421 PRM_DEBUG(memlist); 1422 PRM_DEBUG(memlist_end); 1423 PRM_DEBUG(sysbase); 1424 PRM_DEBUG(syslimit); 1425 1426 kernelheap_init((void *)sysbase, (void *)syslimit, 1427 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1428 1429 /* 1430 * Take the most current snapshot we can by calling mem-update. 1431 */ 1432 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1433 &boot_physavail, &boot_physavail_len, 1434 &boot_virtavail, &boot_virtavail_len); 1435 1436 /* 1437 * Remove the space used by BOP_ALLOC from the kernel heap 1438 * plus the area actually used by the OBP (if any) 1439 * ignoring virtual addresses in virt_avail, above syslimit. 1440 */ 1441 virt_avail = memlist; 1442 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1443 1444 for (cur = virt_avail; cur->next; cur = cur->next) { 1445 uint64_t range_base, range_size; 1446 1447 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1448 continue; 1449 if (range_base >= (uint64_t)syslimit) 1450 break; 1451 /* 1452 * Limit the range to end at syslimit. 1453 */ 1454 range_size = MIN(cur->next->address, 1455 (uint64_t)syslimit) - range_base; 1456 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1457 0, 0, (void *)range_base, (void *)(range_base + range_size), 1458 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1459 } 1460 1461 phys_avail = memlist; 1462 (void) copy_physavail(boot_physavail, boot_physavail_len, 1463 &memlist, 0, 0); 1464 1465 /* 1466 * Add any extra memory after e_text to the phys_avail list, as long 1467 * as there's at least a page to add. 1468 */ 1469 if (extra_etpg) 1470 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1471 &memlist, &phys_avail); 1472 1473 /* 1474 * Add any extra memory after e_data to the phys_avail list as long 1475 * as there's at least a page to add. Usually, there isn't any, 1476 * since extra HME blocks typically get allocated there first before 1477 * using RAM elsewhere. 1478 */ 1479 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1480 nalloc_base = nalloc_end; 1481 ndata_remain_sz = nalloc_end - nalloc_base; 1482 1483 if (ndata_remain_sz >= MMU_PAGESIZE) 1484 memlist_add(va_to_pa(nalloc_base), 1485 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1486 1487 PRM_DEBUG(memlist); 1488 PRM_DEBUG(memlist_sz); 1489 PRM_DEBUG(memspace); 1490 1491 if ((caddr_t)memlist > (memspace + memlist_sz)) 1492 panic("memlist overflow"); 1493 1494 PRM_DEBUG(pp_base); 1495 PRM_DEBUG(memseg_base); 1496 PRM_DEBUG(npages); 1497 1498 /* 1499 * Initialize the page structures from the memory lists. 1500 */ 1501 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1502 1503 availrmem_initial = availrmem = freemem; 1504 PRM_DEBUG(availrmem); 1505 1506 /* 1507 * Some of the locks depend on page_hashsz being set! 1508 * kmem_init() depends on this; so, keep it here. 1509 */ 1510 page_lock_init(); 1511 1512 /* 1513 * Initialize kernel memory allocator. 1514 */ 1515 kmem_init(); 1516 1517 /* 1518 * Initialize bp_mapin(). 1519 */ 1520 bp_init(shm_alignment, HAT_STRICTORDER); 1521 1522 /* 1523 * Reserve space for panicbuf and intr_vector from the 32-bit heap 1524 */ 1525 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1526 panicbuf, panicbuf + PANICBUFSIZE, 1527 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1528 1529 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1530 intr_vector, (caddr_t)intr_vector + IVSIZE, 1531 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1532 1533 mem_config_init(); 1534 } 1535 1536 static void 1537 startup_modules(void) 1538 { 1539 int proplen, nhblk1, nhblk8; 1540 size_t nhblksz; 1541 pgcnt_t hblk_pages, pages_per_hblk; 1542 size_t hme8blk_sz, hme1blk_sz; 1543 1544 /* 1545 * Log any optional messages from the boot program 1546 */ 1547 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1548 if (proplen > 0) { 1549 char *msg; 1550 size_t len = (size_t)proplen; 1551 1552 msg = kmem_zalloc(len, KM_SLEEP); 1553 (void) BOP_GETPROP(bootops, "boot-message", msg); 1554 cmn_err(CE_CONT, "?%s\n", msg); 1555 kmem_free(msg, len); 1556 } 1557 1558 /* 1559 * Let the platforms have a chance to change default 1560 * values before reading system file. 1561 */ 1562 if (&set_platform_defaults) 1563 set_platform_defaults(); 1564 1565 /* 1566 * Calculate default settings of system parameters based upon 1567 * maxusers, yet allow to be overridden via the /etc/system file. 1568 */ 1569 param_calc(0); 1570 1571 mod_setup(); 1572 1573 /* 1574 * If this is a positron, complain and halt. 1575 */ 1576 if (&iam_positron && iam_positron()) { 1577 cmn_err(CE_WARN, "This hardware platform is not supported" 1578 " by this release of Solaris.\n"); 1579 #ifdef DEBUG 1580 prom_enter_mon(); /* Type 'go' to resume */ 1581 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1582 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1583 1584 #else /* DEBUG */ 1585 halt(0); 1586 #endif /* DEBUG */ 1587 } 1588 1589 /* 1590 * If we are running firmware that isn't 64-bit ready 1591 * then complain and halt. 1592 */ 1593 do_prom_version_check(); 1594 1595 /* 1596 * Initialize system parameters 1597 */ 1598 param_init(); 1599 1600 /* 1601 * maxmem is the amount of physical memory we're playing with. 1602 */ 1603 maxmem = physmem; 1604 1605 /* Set segkp limits. */ 1606 ncbase = (caddr_t)SEGDEBUGBASE; 1607 ncend = (caddr_t)SEGDEBUGBASE; 1608 1609 /* 1610 * Initialize the hat layer. 1611 */ 1612 hat_init(); 1613 1614 /* 1615 * Initialize segment management stuff. 1616 */ 1617 seg_init(); 1618 1619 /* 1620 * Create the va>tte handler, so the prom can understand 1621 * kernel translations. The handler is installed later, just 1622 * as we are about to take over the trap table from the prom. 1623 */ 1624 create_va_to_tte(); 1625 1626 /* 1627 * Load the forthdebugger (optional) 1628 */ 1629 forthdebug_init(); 1630 1631 /* 1632 * Create OBP node for console input callbacks 1633 * if it is needed. 1634 */ 1635 startup_create_io_node(); 1636 1637 if (modloadonly("fs", "specfs") == -1) 1638 halt("Can't load specfs"); 1639 1640 if (modloadonly("fs", "devfs") == -1) 1641 halt("Can't load devfs"); 1642 1643 if (modloadonly("misc", "swapgeneric") == -1) 1644 halt("Can't load swapgeneric"); 1645 1646 (void) modloadonly("sys", "lbl_edition"); 1647 1648 dispinit(); 1649 1650 /* 1651 * Infer meanings to the members of the idprom buffer. 1652 */ 1653 parse_idprom(); 1654 1655 /* Read cluster configuration data. */ 1656 clconf_init(); 1657 1658 setup_ddi(); 1659 1660 /* 1661 * Lets take this opportunity to load the root device. 1662 */ 1663 if (loadrootmodules() != 0) 1664 debug_enter("Can't load the root filesystem"); 1665 1666 /* 1667 * Load tod driver module for the tod part found on this system. 1668 * Recompute the cpu frequency/delays based on tod as tod part 1669 * tends to keep time more accurately. 1670 */ 1671 if (&load_tod_module) 1672 load_tod_module(); 1673 1674 /* 1675 * Allow platforms to load modules which might 1676 * be needed after bootops are gone. 1677 */ 1678 if (&load_platform_modules) 1679 load_platform_modules(); 1680 1681 setcpudelay(); 1682 1683 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1684 &boot_physavail, &boot_physavail_len, 1685 &boot_virtavail, &boot_virtavail_len); 1686 1687 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1688 1689 /* 1690 * Calculation and allocation of hmeblks needed to remap 1691 * the memory allocated by PROM till now: 1692 * 1693 * (1) calculate how much virtual memory has been bop_alloc'ed. 1694 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1695 * (3) calculate number of hme8blk's needed to remap this memory 1696 * (4) calculate amount of memory that's consumed by these hme8blk's 1697 * (5) add memory calculated in steps (2) and (4) above. 1698 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1699 * (7) calculate number of hme8blk's needed to remap this memory 1700 * (8) calculate amount of memory that's consumed by these hme8blk's 1701 * (9) allocate additional hme1blk's to hold large mappings. 1702 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1703 * to remap everything with 4M mappings. 1704 * (10) account for partially used hblk8's due to non-64K aligned 1705 * PROM mapping entries. 1706 * (11) add memory calculated in steps (8), (9), and (10) above. 1707 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1708 * is not ready yet, this gets bop_alloc'ed. 1709 * (13) there will be very few bop_alloc's after this point before 1710 * trap table takes over 1711 */ 1712 1713 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1714 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1715 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1716 1717 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1718 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1719 nhblk8 = bop_alloc_pages / pages_per_hblk; 1720 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1721 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1722 bop_alloc_pages += hblk_pages; 1723 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1724 nhblk8 = bop_alloc_pages / pages_per_hblk; 1725 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1726 if (nhblk1 < hblk1_min) 1727 nhblk1 = hblk1_min; 1728 if (nhblk8 < hblk8_min) 1729 nhblk8 = hblk8_min; 1730 1731 /* 1732 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1733 * boundary, the number of hblk8's needed to map the entries in the 1734 * boot_virtavail list needs to be adjusted to take this into 1735 * consideration. Thus, we need to add additional hblk8's since it 1736 * is possible that an hblk8 will not have all 8 slots used due to 1737 * alignment constraints. Since there were boot_virtavail_len entries 1738 * in that list, we need to add that many hblk8's to the number 1739 * already calculated to make sure we don't underestimate. 1740 */ 1741 nhblk8 += boot_virtavail_len; 1742 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1743 1744 /* Allocate in pagesize chunks */ 1745 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1746 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1747 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1748 } 1749 1750 static void 1751 startup_bop_gone(void) 1752 { 1753 extern int bop_io_quiesced; 1754 1755 /* 1756 * Destroy the MD initialized at startup 1757 * The startup initializes the MD framework 1758 * using prom and BOP alloc free it now. 1759 */ 1760 mach_descrip_startup_fini(); 1761 1762 /* 1763 * Call back into boot and release boots resources. 1764 */ 1765 BOP_QUIESCE_IO(bootops); 1766 bop_io_quiesced = 1; 1767 1768 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1769 &boot_physavail, &boot_physavail_len, 1770 &boot_virtavail, &boot_virtavail_len); 1771 /* 1772 * Copy physinstalled list into kernel space. 1773 */ 1774 phys_install = memlist; 1775 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1776 1777 /* 1778 * setup physically contiguous area twice as large as the ecache. 1779 * this is used while doing displacement flush of ecaches 1780 */ 1781 if (&ecache_flush_address) { 1782 ecache_flushaddr = ecache_flush_address(); 1783 if (ecache_flushaddr == (uint64_t)-1) { 1784 cmn_err(CE_PANIC, 1785 "startup: no memory to set ecache_flushaddr"); 1786 } 1787 } 1788 1789 /* 1790 * Virtual available next. 1791 */ 1792 ASSERT(virt_avail != NULL); 1793 memlist_free_list(virt_avail); 1794 virt_avail = memlist; 1795 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1796 1797 /* 1798 * Last chance to ask our booter questions .. 1799 */ 1800 } 1801 1802 1803 /* 1804 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1805 * allocations have been performed. We can't call it in startup_bop_gone 1806 * since later operations can cause obp to allocate more memory. 1807 */ 1808 void 1809 startup_fixup_physavail(void) 1810 { 1811 struct memlist *cur; 1812 1813 /* 1814 * take the most current snapshot we can by calling mem-update 1815 */ 1816 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1817 &boot_physavail, &boot_physavail_len, 1818 &boot_virtavail, &boot_virtavail_len); 1819 1820 /* 1821 * Copy phys_avail list, again. 1822 * Both the kernel/boot and the prom have been allocating 1823 * from the original list we copied earlier. 1824 */ 1825 cur = memlist; 1826 (void) copy_physavail(boot_physavail, boot_physavail_len, 1827 &memlist, 0, 0); 1828 1829 /* 1830 * Add any extra memory after e_text we added to the phys_avail list 1831 * back to the old list. 1832 */ 1833 if (extra_etpg) 1834 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1835 &memlist, &cur); 1836 if (ndata_remain_sz >= MMU_PAGESIZE) 1837 memlist_add(va_to_pa(nalloc_base), 1838 (uint64_t)ndata_remain_sz, &memlist, &cur); 1839 1840 /* 1841 * There isn't any bounds checking on the memlist area 1842 * so ensure it hasn't overgrown. 1843 */ 1844 if ((caddr_t)memlist > (caddr_t)memlist_end) 1845 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1846 1847 /* 1848 * The kernel removes the pages that were allocated for it from 1849 * the freelist, but we now have to find any -extra- pages that 1850 * the prom has allocated for it's own book-keeping, and remove 1851 * them from the freelist too. sigh. 1852 */ 1853 fix_prom_pages(phys_avail, cur); 1854 1855 ASSERT(phys_avail != NULL); 1856 memlist_free_list(phys_avail); 1857 phys_avail = cur; 1858 1859 /* 1860 * We're done with boot. Just after this point in time, boot 1861 * gets unmapped, so we can no longer rely on its services. 1862 * Zero the bootops to indicate this fact. 1863 */ 1864 bootops = (struct bootops *)NULL; 1865 BOOTOPS_GONE(); 1866 } 1867 1868 static void 1869 startup_vm(void) 1870 { 1871 size_t i; 1872 struct segmap_crargs a; 1873 struct segkpm_crargs b; 1874 1875 uint64_t avmem; 1876 caddr_t va; 1877 pgcnt_t max_phys_segkp; 1878 int mnode; 1879 1880 extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg; 1881 1882 /* 1883 * get prom's mappings, create hments for them and switch 1884 * to the kernel context. 1885 */ 1886 hat_kern_setup(); 1887 1888 /* 1889 * Take over trap table 1890 */ 1891 setup_trap_table(); 1892 1893 /* 1894 * Install the va>tte handler, so that the prom can handle 1895 * misses and understand the kernel table layout in case 1896 * we need call into the prom. 1897 */ 1898 install_va_to_tte(); 1899 1900 /* 1901 * Set a flag to indicate that the tba has been taken over. 1902 */ 1903 tba_taken_over = 1; 1904 1905 /* initialize MMU primary context register */ 1906 mmu_init_kcontext(); 1907 1908 /* 1909 * The boot cpu can now take interrupts, x-calls, x-traps 1910 */ 1911 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1912 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1913 1914 /* 1915 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1916 */ 1917 tbr_wr_addr_inited = 1; 1918 1919 /* 1920 * Initialize VM system, and map kernel address space. 1921 */ 1922 kvm_init(); 1923 1924 /* 1925 * XXX4U: previously, we initialized and turned on 1926 * the caches at this point. But of course we have 1927 * nothing to do, as the prom has already done this 1928 * for us -- main memory must be E$able at all times. 1929 */ 1930 1931 /* 1932 * If the following is true, someone has patched 1933 * phsymem to be less than the number of pages that 1934 * the system actually has. Remove pages until system 1935 * memory is limited to the requested amount. Since we 1936 * have allocated page structures for all pages, we 1937 * correct the amount of memory we want to remove 1938 * by the size of the memory used to hold page structures 1939 * for the non-used pages. 1940 */ 1941 if (physmem < npages) { 1942 pgcnt_t diff, off; 1943 struct page *pp; 1944 struct seg kseg; 1945 1946 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1947 1948 off = 0; 1949 diff = npages - physmem; 1950 diff -= mmu_btopr(diff * sizeof (struct page)); 1951 kseg.s_as = &kas; 1952 while (diff--) { 1953 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1954 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1955 &kseg, (caddr_t)off); 1956 if (pp == NULL) 1957 cmn_err(CE_PANIC, "limited physmem too much!"); 1958 page_io_unlock(pp); 1959 page_downgrade(pp); 1960 availrmem--; 1961 off += MMU_PAGESIZE; 1962 } 1963 } 1964 1965 /* 1966 * When printing memory, show the total as physmem less 1967 * that stolen by a debugger. 1968 */ 1969 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1970 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1971 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1972 1973 avmem = (uint64_t)freemem << PAGESHIFT; 1974 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1975 1976 /* For small memory systems disable automatic large pages. */ 1977 if (physmem < auto_lpg_min_physmem) { 1978 exec_lpg_disable = 1; 1979 use_brk_lpg = 0; 1980 use_stk_lpg = 0; 1981 use_zmap_lpg = 0; 1982 } 1983 1984 /* 1985 * Perform platform specific freelist processing 1986 */ 1987 if (&plat_freelist_process) { 1988 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1989 if (mem_node_config[mnode].exists) 1990 plat_freelist_process(mnode); 1991 } 1992 1993 /* 1994 * Initialize the segkp segment type. We position it 1995 * after the configured tables and buffers (whose end 1996 * is given by econtig) and before V_WKBASE_ADDR. 1997 * Also in this area is segkmap (size SEGMAPSIZE). 1998 */ 1999 2000 /* XXX - cache alignment? */ 2001 va = (caddr_t)SEGKPBASE; 2002 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2003 2004 max_phys_segkp = (physmem * 2); 2005 2006 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2007 segkpsize = btop(SEGKPDEFSIZE); 2008 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2009 "segkpsize has been reset to %ld pages", segkpsize); 2010 } 2011 2012 i = ptob(MIN(segkpsize, max_phys_segkp)); 2013 2014 rw_enter(&kas.a_lock, RW_WRITER); 2015 if (seg_attach(&kas, va, i, segkp) < 0) 2016 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2017 if (segkp_create(segkp) != 0) 2018 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2019 rw_exit(&kas.a_lock); 2020 2021 /* 2022 * kpm segment 2023 */ 2024 segmap_kpm = kpm_enable && 2025 segmap_kpm && PAGESIZE == MAXBSIZE; 2026 2027 if (kpm_enable) { 2028 rw_enter(&kas.a_lock, RW_WRITER); 2029 2030 /* 2031 * The segkpm virtual range range is larger than the 2032 * actual physical memory size and also covers gaps in 2033 * the physical address range for the following reasons: 2034 * . keep conversion between segkpm and physical addresses 2035 * simple, cheap and unambiguous. 2036 * . avoid extension/shrink of the the segkpm in case of DR. 2037 * . avoid complexity for handling of virtual addressed 2038 * caches, segkpm and the regular mapping scheme must be 2039 * kept in sync wrt. the virtual color of mapped pages. 2040 * Any accesses to virtual segkpm ranges not backed by 2041 * physical memory will fall through the memseg pfn hash 2042 * and will be handled in segkpm_fault. 2043 * Additional kpm_size spaces needed for vac alias prevention. 2044 */ 2045 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2046 segkpm) < 0) 2047 cmn_err(CE_PANIC, "cannot attach segkpm"); 2048 2049 b.prot = PROT_READ | PROT_WRITE; 2050 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2051 2052 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2053 panic("segkpm_create segkpm"); 2054 2055 rw_exit(&kas.a_lock); 2056 } 2057 2058 /* 2059 * Now create generic mapping segment. This mapping 2060 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2061 * virtual address is greater than the amount of free 2062 * memory that is available, then we trim back the 2063 * segment size to that amount 2064 */ 2065 va = (caddr_t)SEGMAPBASE; 2066 2067 /* 2068 * 1201049: segkmap base address must be MAXBSIZE aligned 2069 */ 2070 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2071 2072 /* 2073 * Set size of segmap to percentage of freemem at boot, 2074 * but stay within the allowable range 2075 * Note we take percentage before converting from pages 2076 * to bytes to avoid an overflow on 32-bit kernels. 2077 */ 2078 i = mmu_ptob((freemem * segmap_percent) / 100); 2079 2080 if (i < MINMAPSIZE) 2081 i = MINMAPSIZE; 2082 2083 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2084 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2085 2086 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2087 2088 rw_enter(&kas.a_lock, RW_WRITER); 2089 if (seg_attach(&kas, va, i, segkmap) < 0) 2090 cmn_err(CE_PANIC, "cannot attach segkmap"); 2091 2092 a.prot = PROT_READ | PROT_WRITE; 2093 a.shmsize = shm_alignment; 2094 a.nfreelist = 0; /* use segmap driver defaults */ 2095 2096 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2097 panic("segmap_create segkmap"); 2098 rw_exit(&kas.a_lock); 2099 2100 segdev_init(); 2101 } 2102 2103 static void 2104 startup_end(void) 2105 { 2106 if ((caddr_t)memlist > (caddr_t)memlist_end) 2107 panic("memlist overflow 2"); 2108 memlist_free_block((caddr_t)memlist, 2109 ((caddr_t)memlist_end - (caddr_t)memlist)); 2110 memlist = NULL; 2111 2112 /* enable page_relocation since OBP is now done */ 2113 page_relocate_ready = 1; 2114 2115 /* 2116 * Perform tasks that get done after most of the VM 2117 * initialization has been done but before the clock 2118 * and other devices get started. 2119 */ 2120 kern_setup1(); 2121 2122 /* 2123 * Intialize the VM arenas for allocating physically 2124 * contiguus memory chunk for interrupt queues snd 2125 * allocate/register boot cpu's queues, if any and 2126 * allocate dump buffer for sun4v systems to store 2127 * extra crash information during crash dump 2128 */ 2129 contig_mem_init(); 2130 mach_descrip_init(); 2131 cpu_intrq_setup(CPU); 2132 cpu_intrq_register(CPU); 2133 mach_htraptrace_setup(CPU->cpu_id); 2134 mach_htraptrace_configure(CPU->cpu_id); 2135 mach_dump_buffer_init(); 2136 2137 /* 2138 * Initialize interrupt related stuff 2139 */ 2140 cpu_intr_alloc(CPU, NINTR_THREADS); 2141 2142 (void) splzs(); /* allow hi clock ints but not zs */ 2143 2144 /* 2145 * Initialize errors. 2146 */ 2147 error_init(); 2148 2149 /* 2150 * Note that we may have already used kernel bcopy before this 2151 * point - but if you really care about this, adb the use_hw_* 2152 * variables to 0 before rebooting. 2153 */ 2154 mach_hw_copy_limit(); 2155 2156 /* 2157 * Install the "real" preemption guards before DDI services 2158 * are available. 2159 */ 2160 (void) prom_set_preprom(kern_preprom); 2161 (void) prom_set_postprom(kern_postprom); 2162 CPU->cpu_m.mutex_ready = 1; 2163 2164 /* 2165 * Initialize segnf (kernel support for non-faulting loads). 2166 */ 2167 segnf_init(); 2168 2169 /* 2170 * Configure the root devinfo node. 2171 */ 2172 configure(); /* set up devices */ 2173 mach_cpu_halt_idle(); 2174 } 2175 2176 2177 void 2178 post_startup(void) 2179 { 2180 #ifdef PTL1_PANIC_DEBUG 2181 extern void init_ptl1_thread(void); 2182 #endif /* PTL1_PANIC_DEBUG */ 2183 extern void abort_sequence_init(void); 2184 2185 /* 2186 * Set the system wide, processor-specific flags to be passed 2187 * to userland via the aux vector for performance hints and 2188 * instruction set extensions. 2189 */ 2190 bind_hwcap(); 2191 2192 /* 2193 * Startup memory scrubber (if any) 2194 */ 2195 mach_memscrub(); 2196 2197 /* 2198 * Allocate soft interrupt to handle abort sequence. 2199 */ 2200 abort_sequence_init(); 2201 2202 /* 2203 * Configure the rest of the system. 2204 * Perform forceloading tasks for /etc/system. 2205 */ 2206 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2207 /* 2208 * ON4.0: Force /proc module in until clock interrupt handle fixed 2209 * ON4.0: This must be fixed or restated in /etc/systems. 2210 */ 2211 (void) modload("fs", "procfs"); 2212 2213 /* load machine class specific drivers */ 2214 load_mach_drivers(); 2215 2216 /* load platform specific drivers */ 2217 if (&load_platform_drivers) 2218 load_platform_drivers(); 2219 2220 /* load vis simulation module, if we are running w/fpu off */ 2221 if (!fpu_exists) { 2222 if (modload("misc", "vis") == -1) 2223 halt("Can't load vis"); 2224 } 2225 2226 mach_fpras(); 2227 2228 maxmem = freemem; 2229 2230 #ifdef PTL1_PANIC_DEBUG 2231 init_ptl1_thread(); 2232 #endif /* PTL1_PANIC_DEBUG */ 2233 2234 if (&cif_init) 2235 cif_init(); 2236 } 2237 2238 #ifdef PTL1_PANIC_DEBUG 2239 int ptl1_panic_test = 0; 2240 int ptl1_panic_xc_one_test = 0; 2241 int ptl1_panic_xc_all_test = 0; 2242 int ptl1_panic_xt_one_test = 0; 2243 int ptl1_panic_xt_all_test = 0; 2244 kthread_id_t ptl1_thread_p = NULL; 2245 kcondvar_t ptl1_cv; 2246 kmutex_t ptl1_mutex; 2247 int ptl1_recurse_count_threshold = 0x40; 2248 int ptl1_recurse_trap_threshold = 0x3d; 2249 extern void ptl1_recurse(int, int); 2250 extern void ptl1_panic_xt(int, int); 2251 2252 /* 2253 * Called once per second by timeout() to wake up 2254 * the ptl1_panic thread to see if it should cause 2255 * a trap to the ptl1_panic() code. 2256 */ 2257 /* ARGSUSED */ 2258 static void 2259 ptl1_wakeup(void *arg) 2260 { 2261 mutex_enter(&ptl1_mutex); 2262 cv_signal(&ptl1_cv); 2263 mutex_exit(&ptl1_mutex); 2264 } 2265 2266 /* 2267 * ptl1_panic cross call function: 2268 * Needed because xc_one() and xc_some() can pass 2269 * 64 bit args but ptl1_recurse() expects ints. 2270 */ 2271 static void 2272 ptl1_panic_xc(void) 2273 { 2274 ptl1_recurse(ptl1_recurse_count_threshold, 2275 ptl1_recurse_trap_threshold); 2276 } 2277 2278 /* 2279 * The ptl1 thread waits for a global flag to be set 2280 * and uses the recurse thresholds to set the stack depth 2281 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2282 * or indirectly via the cross call and cross trap functions. 2283 * 2284 * This is useful testing stack overflows and normal 2285 * ptl1_panic() states with a know stack frame. 2286 * 2287 * ptl1_recurse() is an asm function in ptl1_panic.s that 2288 * sets the {In, Local, Out, and Global} registers to a 2289 * know state on the stack and just prior to causing a 2290 * test ptl1_panic trap. 2291 */ 2292 static void 2293 ptl1_thread(void) 2294 { 2295 mutex_enter(&ptl1_mutex); 2296 while (ptl1_thread_p) { 2297 cpuset_t other_cpus; 2298 int cpu_id; 2299 int my_cpu_id; 2300 int target_cpu_id; 2301 int target_found; 2302 2303 if (ptl1_panic_test) { 2304 ptl1_recurse(ptl1_recurse_count_threshold, 2305 ptl1_recurse_trap_threshold); 2306 } 2307 2308 /* 2309 * Find potential targets for x-call and x-trap, 2310 * if any exist while preempt is disabled we 2311 * start a ptl1_panic if requested via a 2312 * globals. 2313 */ 2314 kpreempt_disable(); 2315 my_cpu_id = CPU->cpu_id; 2316 other_cpus = cpu_ready_set; 2317 CPUSET_DEL(other_cpus, CPU->cpu_id); 2318 target_found = 0; 2319 if (!CPUSET_ISNULL(other_cpus)) { 2320 /* 2321 * Pick the first one 2322 */ 2323 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2324 if (cpu_id == my_cpu_id) 2325 continue; 2326 2327 if (CPU_XCALL_READY(cpu_id)) { 2328 target_cpu_id = cpu_id; 2329 target_found = 1; 2330 break; 2331 } 2332 } 2333 ASSERT(target_found); 2334 2335 if (ptl1_panic_xc_one_test) { 2336 xc_one(target_cpu_id, 2337 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2338 } 2339 if (ptl1_panic_xc_all_test) { 2340 xc_some(other_cpus, 2341 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2342 } 2343 if (ptl1_panic_xt_one_test) { 2344 xt_one(target_cpu_id, 2345 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2346 } 2347 if (ptl1_panic_xt_all_test) { 2348 xt_some(other_cpus, 2349 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2350 } 2351 } 2352 kpreempt_enable(); 2353 (void) timeout(ptl1_wakeup, NULL, hz); 2354 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2355 } 2356 mutex_exit(&ptl1_mutex); 2357 } 2358 2359 /* 2360 * Called during early startup to create the ptl1_thread 2361 */ 2362 void 2363 init_ptl1_thread(void) 2364 { 2365 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2366 &p0, TS_RUN, 0); 2367 } 2368 #endif /* PTL1_PANIC_DEBUG */ 2369 2370 2371 /* 2372 * Add to a memory list. 2373 * start = start of new memory segment 2374 * len = length of new memory segment in bytes 2375 * memlistp = pointer to array of available memory segment structures 2376 * curmemlistp = memory list to which to add segment. 2377 */ 2378 static void 2379 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2380 struct memlist **curmemlistp) 2381 { 2382 struct memlist *new; 2383 2384 new = *memlistp; 2385 new->address = start; 2386 new->size = len; 2387 *memlistp = new + 1; 2388 2389 memlist_insert(new, curmemlistp); 2390 } 2391 2392 /* 2393 * In the case of architectures that support dynamic addition of 2394 * memory at run-time there are two cases where memsegs need to 2395 * be initialized and added to the memseg list. 2396 * 1) memsegs that are constructed at startup. 2397 * 2) memsegs that are constructed at run-time on 2398 * hot-plug capable architectures. 2399 * This code was originally part of the function kphysm_init(). 2400 */ 2401 2402 static void 2403 memseg_list_add(struct memseg *memsegp) 2404 { 2405 struct memseg **prev_memsegp; 2406 pgcnt_t num; 2407 2408 /* insert in memseg list, decreasing number of pages order */ 2409 2410 num = MSEG_NPAGES(memsegp); 2411 2412 for (prev_memsegp = &memsegs; *prev_memsegp; 2413 prev_memsegp = &((*prev_memsegp)->next)) { 2414 if (num > MSEG_NPAGES(*prev_memsegp)) 2415 break; 2416 } 2417 2418 memsegp->next = *prev_memsegp; 2419 *prev_memsegp = memsegp; 2420 2421 if (kpm_enable) { 2422 memsegp->nextpa = (memsegp->next) ? 2423 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2424 2425 if (prev_memsegp != &memsegs) { 2426 struct memseg *msp; 2427 msp = (struct memseg *)((caddr_t)prev_memsegp - 2428 offsetof(struct memseg, next)); 2429 msp->nextpa = va_to_pa(memsegp); 2430 } else { 2431 memsegspa = va_to_pa(memsegs); 2432 } 2433 } 2434 } 2435 2436 /* 2437 * PSM add_physmem_cb(). US-II and newer processors have some 2438 * flavor of the prefetch capability implemented. We exploit 2439 * this capability for optimum performance. 2440 */ 2441 #define PREFETCH_BYTES 64 2442 2443 void 2444 add_physmem_cb(page_t *pp, pfn_t pnum) 2445 { 2446 extern void prefetch_page_w(void *); 2447 2448 pp->p_pagenum = pnum; 2449 2450 /* 2451 * Prefetch one more page_t into E$. To prevent future 2452 * mishaps with the sizeof(page_t) changing on us, we 2453 * catch this on debug kernels if we can't bring in the 2454 * entire hpage with 2 PREFETCH_BYTES reads. See 2455 * also, sun4u/cpu/cpu_module.c 2456 */ 2457 /*LINTED*/ 2458 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2459 prefetch_page_w((char *)pp); 2460 } 2461 2462 /* 2463 * kphysm_init() tackles the problem of initializing physical memory. 2464 * The old startup made some assumptions about the kernel living in 2465 * physically contiguous space which is no longer valid. 2466 */ 2467 static void 2468 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2469 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2470 { 2471 struct memlist *pmem; 2472 struct memseg *msp; 2473 pfn_t base; 2474 pgcnt_t num; 2475 pfn_t lastseg_pages_end = 0; 2476 pgcnt_t nelem_used = 0; 2477 2478 ASSERT(page_hash != NULL && page_hashsz != 0); 2479 2480 msp = memsegp; 2481 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2482 2483 /* 2484 * Build the memsegs entry 2485 */ 2486 num = btop(pmem->size); 2487 if (num > npages) 2488 num = npages; 2489 npages -= num; 2490 base = btop(pmem->address); 2491 2492 msp->pages = pp; 2493 msp->epages = pp + num; 2494 msp->pages_base = base; 2495 msp->pages_end = base + num; 2496 2497 if (kpm_enable) { 2498 pfn_t pbase_a; 2499 pfn_t pend_a; 2500 pfn_t prev_pend_a; 2501 pgcnt_t nelem; 2502 2503 msp->pagespa = va_to_pa(pp); 2504 msp->epagespa = va_to_pa(pp + num); 2505 pbase_a = kpmptop(ptokpmp(base)); 2506 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2507 nelem = ptokpmp(pend_a - pbase_a); 2508 msp->kpm_nkpmpgs = nelem; 2509 msp->kpm_pbase = pbase_a; 2510 if (lastseg_pages_end) { 2511 /* 2512 * Assume phys_avail is in ascending order 2513 * of physical addresses. 2514 */ 2515 ASSERT(base + num > lastseg_pages_end); 2516 prev_pend_a = kpmptop( 2517 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2518 2519 if (prev_pend_a > pbase_a) { 2520 /* 2521 * Overlap, more than one memseg may 2522 * point to the same kpm_page range. 2523 */ 2524 if (kpm_smallpages == 0) { 2525 msp->kpm_pages = 2526 (kpm_page_t *)kpm_pp - 1; 2527 kpm_pp = (uintptr_t) 2528 ((kpm_page_t *)kpm_pp 2529 + nelem - 1); 2530 } else { 2531 msp->kpm_spages = 2532 (kpm_spage_t *)kpm_pp - 1; 2533 kpm_pp = (uintptr_t) 2534 ((kpm_spage_t *)kpm_pp 2535 + nelem - 1); 2536 } 2537 nelem_used += nelem - 1; 2538 2539 } else { 2540 if (kpm_smallpages == 0) { 2541 msp->kpm_pages = 2542 (kpm_page_t *)kpm_pp; 2543 kpm_pp = (uintptr_t) 2544 ((kpm_page_t *)kpm_pp 2545 + nelem); 2546 } else { 2547 msp->kpm_spages = 2548 (kpm_spage_t *)kpm_pp; 2549 kpm_pp = (uintptr_t) 2550 ((kpm_spage_t *) 2551 kpm_pp + nelem); 2552 } 2553 nelem_used += nelem; 2554 } 2555 2556 } else { 2557 if (kpm_smallpages == 0) { 2558 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2559 kpm_pp = (uintptr_t) 2560 ((kpm_page_t *)kpm_pp + nelem); 2561 } else { 2562 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2563 kpm_pp = (uintptr_t) 2564 ((kpm_spage_t *)kpm_pp + nelem); 2565 } 2566 nelem_used = nelem; 2567 } 2568 2569 if (nelem_used > kpm_npages) 2570 panic("kphysm_init: kpm_pp overflow\n"); 2571 2572 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2573 lastseg_pages_end = msp->pages_end; 2574 } 2575 2576 memseg_list_add(msp); 2577 2578 /* 2579 * add_physmem() initializes the PSM part of the page 2580 * struct by calling the PSM back with add_physmem_cb(). 2581 * In addition it coalesces pages into larger pages as 2582 * it initializes them. 2583 */ 2584 add_physmem(pp, num, base); 2585 pp += num; 2586 msp++; 2587 } 2588 2589 build_pfn_hash(); 2590 } 2591 2592 /* 2593 * Kernel VM initialization. 2594 * Assumptions about kernel address space ordering: 2595 * (1) gap (user space) 2596 * (2) kernel text 2597 * (3) kernel data/bss 2598 * (4) gap 2599 * (5) kernel data structures 2600 * (6) gap 2601 * (7) debugger (optional) 2602 * (8) monitor 2603 * (9) gap (possibly null) 2604 * (10) dvma 2605 * (11) devices 2606 */ 2607 static void 2608 kvm_init(void) 2609 { 2610 /* 2611 * Put the kernel segments in kernel address space. 2612 */ 2613 rw_enter(&kas.a_lock, RW_WRITER); 2614 as_avlinit(&kas); 2615 2616 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2617 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2618 (void) segkmem_create(&ktextseg); 2619 2620 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2621 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2622 (void) segkmem_create(&ktexthole); 2623 2624 (void) seg_attach(&kas, (caddr_t)valloc_base, 2625 (size_t)(econtig32 - valloc_base), &kvalloc); 2626 (void) segkmem_create(&kvalloc); 2627 2628 if (kmem64_base) { 2629 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2630 (size_t)(kmem64_end - kmem64_base), &kmem64); 2631 (void) segkmem_create(&kmem64); 2632 } 2633 2634 /* 2635 * We're about to map out /boot. This is the beginning of the 2636 * system resource management transition. We can no longer 2637 * call into /boot for I/O or memory allocations. 2638 */ 2639 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2640 (void) segkmem_create(&kvseg); 2641 hblk_alloc_dynamic = 1; 2642 2643 /* 2644 * we need to preallocate pages for DR operations before enabling large 2645 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2646 */ 2647 memseg_remap_init(); 2648 2649 /* at this point we are ready to use large page heap */ 2650 segkmem_heap_lp_init(); 2651 2652 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2653 &kvseg32); 2654 (void) segkmem_create(&kvseg32); 2655 2656 /* 2657 * Create a segment for the debugger. 2658 */ 2659 (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2660 &kdebugseg); 2661 (void) segkmem_create(&kdebugseg); 2662 2663 rw_exit(&kas.a_lock); 2664 } 2665 2666 char obp_tte_str[] = 2667 "h# %x constant MMU_PAGESHIFT " 2668 "h# %x constant TTE8K " 2669 "h# %x constant SFHME_SIZE " 2670 "h# %x constant SFHME_TTE " 2671 "h# %x constant HMEBLK_TAG " 2672 "h# %x constant HMEBLK_NEXT " 2673 "h# %x constant HMEBLK_MISC " 2674 "h# %x constant HMEBLK_HME1 " 2675 "h# %x constant NHMENTS " 2676 "h# %x constant HBLK_SZMASK " 2677 "h# %x constant HBLK_RANGE_SHIFT " 2678 "h# %x constant HMEBP_HBLK " 2679 "h# %x constant HMEBUCKET_SIZE " 2680 "h# %x constant HTAG_SFMMUPSZ " 2681 "h# %x constant HTAG_REHASHSZ " 2682 "h# %x constant mmu_hashcnt " 2683 "h# %p constant uhme_hash " 2684 "h# %p constant khme_hash " 2685 "h# %x constant UHMEHASH_SZ " 2686 "h# %x constant KHMEHASH_SZ " 2687 "h# %p constant KHATID " 2688 "h# %x constant CTX_SIZE " 2689 "h# %x constant CTX_SFMMU " 2690 "h# %p constant ctxs " 2691 "h# %x constant ASI_MEM " 2692 2693 ": PHYS-X@ ( phys -- data ) " 2694 " ASI_MEM spacex@ " 2695 "; " 2696 2697 ": PHYS-W@ ( phys -- data ) " 2698 " ASI_MEM spacew@ " 2699 "; " 2700 2701 ": PHYS-L@ ( phys -- data ) " 2702 " ASI_MEM spaceL@ " 2703 "; " 2704 2705 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2706 " 3 * MMU_PAGESHIFT + " 2707 "; " 2708 2709 ": TTE_IS_VALID ( ttep -- flag ) " 2710 " PHYS-X@ 0< " 2711 "; " 2712 2713 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2714 " dup TTE8K = if " 2715 " drop HBLK_RANGE_SHIFT " 2716 " else " 2717 " TTE_PAGE_SHIFT " 2718 " then " 2719 "; " 2720 2721 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2722 " tuck >> swap MMU_PAGESHIFT - << " 2723 "; " 2724 2725 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2726 " >> over xor swap ( hash sfmmup ) " 2727 " KHATID <> if ( hash ) " 2728 " UHMEHASH_SZ and ( bucket ) " 2729 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2730 " else ( hash ) " 2731 " KHMEHASH_SZ and ( bucket ) " 2732 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2733 " then ( hmebp ) " 2734 "; " 2735 2736 ": HME_HASH_TABLE_SEARCH " 2737 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2738 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2739 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2740 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2741 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2742 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2743 " else " 2744 " hmeblk_next + phys-x@ false " 2745 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2746 " then " 2747 " else " 2748 " hmeblk_next + phys-x@ false " 2749 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2750 " then " 2751 " else " 2752 " true " 2753 " then " 2754 " until r> drop " 2755 "; " 2756 2757 ": CNUM_TO_SFMMUP ( cnum -- sfmmup ) " 2758 " CTX_SIZE * ctxs + CTX_SFMMU + " 2759 "x@ " 2760 "; " 2761 2762 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2763 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2764 " HTAG_REHASHSZ << or nip ( hblktag ) " 2765 "; " 2766 2767 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2768 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2769 " TTE8K = if ( hmeblkp addr ) " 2770 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2771 " else ( hmeblkp addr ) " 2772 " drop 0 ( hmeblkp 0 ) " 2773 " then ( hmeblkp hme-index ) " 2774 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2775 " SFHME_TTE + ( ttep ) " 2776 "; " 2777 2778 ": unix-tte ( addr cnum -- false | tte-data true ) " 2779 " CNUM_TO_SFMMUP ( addr sfmmup ) " 2780 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2781 " 2dup swap i HME_HASH_SHIFT " 2782 "( addr sfmmup sfmmup addr hmeshift ) " 2783 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2784 " over i 4 pick " 2785 "( addr sfmmup hmebp sfmmup rehash addr ) " 2786 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2787 " HME_HASH_TABLE_SEARCH " 2788 "( addr sfmmup { null | hmeblkp } ) " 2789 " ?dup if ( addr sfmmup hmeblkp ) " 2790 " nip swap HBLK_TO_TTEP ( ttep ) " 2791 " dup TTE_IS_VALID if ( valid-ttep ) " 2792 " PHYS-X@ true ( tte-data true ) " 2793 " else ( invalid-tte ) " 2794 " drop false ( false ) " 2795 " then ( false | tte-data true ) " 2796 " unloop exit ( false | tte-data true ) " 2797 " then ( addr sfmmup ) " 2798 " loop ( addr sfmmup ) " 2799 " 2drop false ( false ) " 2800 "; " 2801 ; 2802 2803 void 2804 create_va_to_tte(void) 2805 { 2806 char *bp; 2807 extern int khmehash_num, uhmehash_num; 2808 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2809 2810 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2811 2812 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2813 2814 /* 2815 * Teach obp how to parse our sw ttes. 2816 */ 2817 (void) sprintf(bp, obp_tte_str, 2818 MMU_PAGESHIFT, 2819 TTE8K, 2820 sizeof (struct sf_hment), 2821 OFFSET(struct sf_hment, hme_tte), 2822 OFFSET(struct hme_blk, hblk_tag), 2823 OFFSET(struct hme_blk, hblk_nextpa), 2824 OFFSET(struct hme_blk, hblk_misc), 2825 OFFSET(struct hme_blk, hblk_hme), 2826 NHMENTS, 2827 HBLK_SZMASK, 2828 HBLK_RANGE_SHIFT, 2829 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2830 sizeof (struct hmehash_bucket), 2831 HTAG_SFMMUPSZ, 2832 HTAG_REHASHSZ, 2833 mmu_hashcnt, 2834 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2835 (caddr_t)va_to_pa((caddr_t)khme_hash), 2836 UHMEHASH_SZ, 2837 KHMEHASH_SZ, 2838 KHATID, 2839 sizeof (struct ctx), 2840 OFFSET(struct ctx, ctx_sfmmu), 2841 ctxs, 2842 ASI_MEM); 2843 prom_interpret(bp, 0, 0, 0, 0, 0); 2844 2845 kobj_free(bp, MMU_PAGESIZE); 2846 } 2847 2848 void 2849 install_va_to_tte(void) 2850 { 2851 /* 2852 * advise prom that he can use unix-tte 2853 */ 2854 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2855 } 2856 2857 2858 /* 2859 * Because kmdb links prom_stdout_is_framebuffer into its own 2860 * module, we add "device-type=display" here for /os-io node, so that 2861 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 2862 * is registered into OBP. 2863 */ 2864 static char *create_node = 2865 "\" /\" find-device " 2866 "new-device " 2867 "\" os-io\" device-name " 2868 "\" display\" device-type " 2869 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2870 " 2>r swap 2 2r> ['] $callback catch if " 2871 " 2drop 3drop 0 " 2872 " then " 2873 "; " 2874 ": read ( adr,len -- #read ) " 2875 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2876 " ( retN ... ret1 N ) " 2877 " ?dup if " 2878 " swap >r 1- 0 ?do drop loop r> " 2879 " else " 2880 " -2 " 2881 " then " 2882 "; " 2883 ": write ( adr,len -- #written ) " 2884 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2885 " ( retN ... ret1 N ) " 2886 " ?dup if " 2887 " swap >r 1- 0 ?do drop loop r> " 2888 " else " 2889 " 0 " 2890 " then " 2891 "; " 2892 ": poll-tty ( -- ) ; " 2893 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2894 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2895 ": cb-give/take ( $method -- ) " 2896 " 0 -rot ['] $callback catch ?dup if " 2897 " >r 2drop 2drop r> throw " 2898 " else " 2899 " 0 ?do drop loop " 2900 " then " 2901 "; " 2902 ": give ( -- ) \" exit-input\" cb-give/take ; " 2903 ": take ( -- ) \" enter-input\" cb-give/take ; " 2904 ": open ( -- ok? ) true ; " 2905 ": close ( -- ) ; " 2906 "finish-device " 2907 "device-end "; 2908 2909 /* 2910 * Create the OBP input/output node (FCode serial driver). 2911 * It is needed for both USB console keyboard and for 2912 * the kernel terminal emulator. It is too early to check for a 2913 * kernel console compatible framebuffer now, so we create this 2914 * so that we're ready if we need to enable kernel terminal emulation. 2915 * 2916 * When the USB software takes over the input device at the time 2917 * consconfig runs, OBP's stdin is redirected to this node. 2918 * Whenever the FORTH user interface is used after this switch, 2919 * the node will call back into the kernel for console input. 2920 * If a serial device such as ttya or a UART with a Type 5 keyboard 2921 * attached is used, OBP takes over the serial device when the system 2922 * goes to the debugger after the system is booted. This sharing 2923 * of the relatively simple serial device is difficult but possible. 2924 * Sharing the USB host controller is impossible due its complexity. 2925 * 2926 * Similarly to USB keyboard input redirection, after consconfig_dacf 2927 * configures a kernel console framebuffer as the standard output 2928 * device, OBP's stdout is switched to to vector through the 2929 * /os-io node into the kernel terminal emulator. 2930 */ 2931 static void 2932 startup_create_io_node(void) 2933 { 2934 prom_interpret(create_node, 0, 0, 0, 0, 0); 2935 } 2936 2937 2938 static void 2939 do_prom_version_check(void) 2940 { 2941 int i; 2942 pnode_t node; 2943 char buf[64]; 2944 static char drev[] = "Down-rev firmware detected%s\n" 2945 "\tPlease upgrade to the following minimum version:\n" 2946 "\t\t%s\n"; 2947 2948 i = prom_version_check(buf, sizeof (buf), &node); 2949 2950 if (i == PROM_VER64_OK) 2951 return; 2952 2953 if (i == PROM_VER64_UPGRADE) { 2954 cmn_err(CE_WARN, drev, "", buf); 2955 2956 #ifdef DEBUG 2957 prom_enter_mon(); /* Type 'go' to continue */ 2958 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2959 return; 2960 #else 2961 halt(0); 2962 #endif 2963 } 2964 2965 /* 2966 * The other possibility is that this is a server running 2967 * good firmware, but down-rev firmware was detected on at 2968 * least one other cpu board. We just complain if we see 2969 * that. 2970 */ 2971 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2972 } 2973 2974 static void 2975 kpm_init() 2976 { 2977 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 2978 kpm_pgsz = 1ull << kpm_pgshft; 2979 kpm_pgoff = kpm_pgsz - 1; 2980 kpmp2pshft = kpm_pgshft - PAGESHIFT; 2981 kpmpnpgs = 1 << kpmp2pshft; 2982 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 2983 } 2984 2985 void 2986 kpm_npages_setup(int memblocks) 2987 { 2988 /* 2989 * npages can be scattered in a maximum of 'memblocks' 2990 */ 2991 kpm_npages = ptokpmpr(npages) + memblocks; 2992 } 2993 2994 /* 2995 * Must be defined in platform dependent code. 2996 */ 2997 extern caddr_t modtext; 2998 extern size_t modtext_sz; 2999 extern caddr_t moddata; 3000 3001 #define HEAPTEXT_ARENA(addr) \ 3002 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3003 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3004 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3005 3006 #define HEAPTEXT_OVERSIZED(addr) \ 3007 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3008 3009 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3010 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3011 kmutex_t texthole_lock; 3012 3013 char kern_bootargs[OBP_MAXPATHLEN]; 3014 3015 void 3016 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3017 { 3018 uintptr_t addr, limit; 3019 3020 addr = HEAPTEXT_BASE; 3021 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3022 3023 /* 3024 * Before we initialize the text_arena, we want to punch holes in the 3025 * underlying heaptext_arena. This guarantees that for any text 3026 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3027 */ 3028 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3029 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3030 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3031 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3032 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3033 } 3034 3035 /* 3036 * Allocate one page at the oversize to break up the text region 3037 * from the oversized region. 3038 */ 3039 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3040 (void *)limit, (void *)(limit + PAGESIZE), 3041 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3042 3043 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3044 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3045 heaptext_arena, 0, VM_SLEEP); 3046 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3047 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3048 } 3049 3050 caddr_t 3051 kobj_text_alloc(vmem_t *arena, size_t size) 3052 { 3053 caddr_t rval, better; 3054 3055 /* 3056 * First, try a sleeping allocation. 3057 */ 3058 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3059 3060 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3061 return (rval); 3062 3063 /* 3064 * We didn't get the area that we wanted. We're going to try to do an 3065 * allocation with explicit constraints. 3066 */ 3067 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3068 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3069 VM_NOSLEEP | VM_BESTFIT); 3070 3071 if (better != NULL) { 3072 /* 3073 * That worked. Free our first attempt and return. 3074 */ 3075 vmem_free(arena, rval, size); 3076 return (better); 3077 } 3078 3079 /* 3080 * That didn't work; we'll have to return our first attempt. 3081 */ 3082 return (rval); 3083 } 3084 3085 caddr_t 3086 kobj_texthole_alloc(caddr_t addr, size_t size) 3087 { 3088 int arena = HEAPTEXT_ARENA(addr); 3089 char c[30]; 3090 uintptr_t base; 3091 3092 if (HEAPTEXT_OVERSIZED(addr)) { 3093 /* 3094 * If this is an oversized allocation, there is no text hole 3095 * available for it; return NULL. 3096 */ 3097 return (NULL); 3098 } 3099 3100 mutex_enter(&texthole_lock); 3101 3102 if (texthole_arena[arena] == NULL) { 3103 ASSERT(texthole_source[arena] == NULL); 3104 3105 if (arena == 0) { 3106 texthole_source[0] = vmem_create("module_text_holesrc", 3107 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3108 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3109 0, VM_SLEEP); 3110 } else { 3111 base = HEAPTEXT_BASE + 3112 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3113 3114 (void) snprintf(c, sizeof (c), 3115 "heaptext_holesrc_%d", arena); 3116 3117 texthole_source[arena] = vmem_create(c, (void *)base, 3118 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3119 0, VM_SLEEP); 3120 } 3121 3122 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3123 3124 texthole_arena[arena] = vmem_create(c, NULL, 0, 3125 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3126 texthole_source[arena], 0, VM_SLEEP); 3127 } 3128 3129 mutex_exit(&texthole_lock); 3130 3131 ASSERT(texthole_arena[arena] != NULL); 3132 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3133 return (vmem_alloc(texthole_arena[arena], size, 3134 VM_BESTFIT | VM_NOSLEEP)); 3135 } 3136 3137 void 3138 kobj_texthole_free(caddr_t addr, size_t size) 3139 { 3140 int arena = HEAPTEXT_ARENA(addr); 3141 3142 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3143 ASSERT(texthole_arena[arena] != NULL); 3144 vmem_free(texthole_arena[arena], addr, size); 3145 } 3146