1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern void cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern void mach_dump_buffer_init(void); 74 extern void mach_descrip_init(void); 75 extern void mach_descrip_startup_fini(void); 76 extern void mach_memscrub(void); 77 extern void mach_fpras(void); 78 extern void mach_cpu_halt_idle(void); 79 extern void mach_hw_copy_limit(void); 80 extern void load_mach_drivers(void); 81 extern void load_tod_module(void); 82 #pragma weak load_tod_module 83 84 extern int ndata_alloc_mmfsa(struct memlist *ndata); 85 #pragma weak ndata_alloc_mmfsa 86 87 extern void cif_init(void); 88 #pragma weak cif_init 89 90 extern void parse_idprom(void); 91 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 92 extern void mem_config_init(void); 93 extern void memseg_remap_init(void); 94 95 extern void mach_kpm_init(void); 96 97 /* 98 * External Data: 99 */ 100 extern int vac_size; /* cache size in bytes */ 101 extern uint_t vac_mask; /* VAC alignment consistency mask */ 102 extern uint_t vac_colors; 103 104 /* 105 * Global Data Definitions: 106 */ 107 108 /* 109 * XXX - Don't port this to new architectures 110 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 111 * 'romp' has no use with a prom with an IEEE 1275 client interface. 112 * The driver doesn't use the value, but it depends on the symbol. 113 */ 114 void *romp; /* veritas driver won't load without romp 4154976 */ 115 /* 116 * Declare these as initialized data so we can patch them. 117 */ 118 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 119 pgcnt_t segkpsize = 120 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 121 uint_t segmap_percent = 12; /* Size of segmap segment */ 122 123 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 124 int vac_copyback = 1; 125 char *cache_mode = NULL; 126 int use_mix = 1; 127 int prom_debug = 0; 128 129 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 caddr_t sdata; /* beginning of data segment */ 151 152 caddr_t extra_etva; /* beginning of unused nucleus text */ 153 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 154 155 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 156 caddr_t nalloc_base; /* beginning of nucleus allocation */ 157 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 158 caddr_t valloc_base; /* beginning of kvalloc segment */ 159 160 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 161 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 162 163 uintptr_t shm_alignment; /* VAC address consistency modulus */ 164 struct memlist *phys_install; /* Total installed physical memory */ 165 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 166 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 167 struct memlist ndata; /* memlist of nucleus allocatable memory */ 168 int memexp_flag; /* memory expansion card flag */ 169 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 170 pgcnt_t obp_pages; /* Physical pages used by OBP */ 171 172 /* 173 * VM data structures 174 */ 175 long page_hashsz; /* Size of page hash table (power of two) */ 176 struct page *pp_base; /* Base of system page struct array */ 177 size_t pp_sz; /* Size in bytes of page struct array */ 178 struct page **page_hash; /* Page hash table */ 179 struct seg ktextseg; /* Segment used for kernel executable image */ 180 struct seg kvalloc; /* Segment used for "valloc" mapping */ 181 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 182 struct seg ktexthole; /* Segment used for nucleus text hole */ 183 struct seg kmapseg; /* Segment used for generic kernel mappings */ 184 struct seg kpmseg; /* Segment used for physical mapping */ 185 struct seg kdebugseg; /* Segment used for the kernel debugger */ 186 187 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 188 size_t kpm_pp_sz; /* Size of system kpm_page array */ 189 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 190 191 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 192 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 193 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 194 195 int segzio_fromheap = 0; /* zio allocations occur from heap */ 196 caddr_t segzio_base; /* Base address of segzio */ 197 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 198 199 /* 200 * debugger pages (if allocated) 201 */ 202 struct vnode kdebugvp; 203 204 /* 205 * Segment for relocated kernel structures in 64-bit large RAM kernels 206 */ 207 struct seg kmem64; 208 209 struct memseg *memseg_base; 210 size_t memseg_sz; /* Used to translate a va to page */ 211 struct vnode unused_pages_vp; 212 213 /* 214 * VM data structures allocated early during boot. 215 */ 216 size_t pagehash_sz; 217 uint64_t memlist_sz; 218 219 char tbr_wr_addr_inited = 0; 220 221 222 /* 223 * Static Routines: 224 */ 225 static void memlist_add(uint64_t, uint64_t, struct memlist **, 226 struct memlist **); 227 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 228 pgcnt_t); 229 static void kvm_init(void); 230 231 static void startup_init(void); 232 static void startup_memlist(void); 233 static void startup_modules(void); 234 static void startup_bop_gone(void); 235 static void startup_vm(void); 236 static void startup_end(void); 237 static void setup_cage_params(void); 238 static void startup_create_io_node(void); 239 240 static pgcnt_t npages; 241 static struct memlist *memlist; 242 void *memlist_end; 243 244 static pgcnt_t bop_alloc_pages; 245 static caddr_t hblk_base; 246 uint_t hblk_alloc_dynamic = 0; 247 uint_t hblk1_min = H1MIN; 248 uint_t hblk8_min; 249 250 251 /* 252 * Hooks for unsupported platforms and down-rev firmware 253 */ 254 int iam_positron(void); 255 #pragma weak iam_positron 256 static void do_prom_version_check(void); 257 static void kpm_init(void); 258 static void kpm_npages_setup(int); 259 static void kpm_memseg_init(void); 260 261 /* 262 * After receiving a thermal interrupt, this is the number of seconds 263 * to delay before shutting off the system, assuming 264 * shutdown fails. Use /etc/system to change the delay if this isn't 265 * large enough. 266 */ 267 int thermal_powerdown_delay = 1200; 268 269 /* 270 * Used to hold off page relocations into the cage until OBP has completed 271 * its boot-time handoff of its resources to the kernel. 272 */ 273 int page_relocate_ready = 0; 274 275 /* 276 * Enable some debugging messages concerning memory usage... 277 */ 278 #ifdef DEBUGGING_MEM 279 static int debugging_mem; 280 static void 281 printmemlist(char *title, struct memlist *list) 282 { 283 if (!debugging_mem) 284 return; 285 286 printf("%s\n", title); 287 288 while (list) { 289 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 290 (uint32_t)(list->address >> 32), (uint32_t)list->address, 291 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 292 list = list->next; 293 } 294 } 295 296 void 297 printmemseg(struct memseg *memseg) 298 { 299 if (!debugging_mem) 300 return; 301 302 printf("memseg\n"); 303 304 while (memseg) { 305 prom_printf("\tpage = 0x%p, epage = 0x%p, " 306 "pfn = 0x%x, epfn = 0x%x\n", 307 memseg->pages, memseg->epages, 308 memseg->pages_base, memseg->pages_end); 309 memseg = memseg->next; 310 } 311 } 312 313 #define debug_pause(str) halt((str)) 314 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 315 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 316 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 317 #define MPRINTF3(str, a, b, c) \ 318 if (debugging_mem) prom_printf((str), (a), (b), (c)) 319 #else /* DEBUGGING_MEM */ 320 #define MPRINTF(str) 321 #define MPRINTF1(str, a) 322 #define MPRINTF2(str, a, b) 323 #define MPRINTF3(str, a, b, c) 324 #endif /* DEBUGGING_MEM */ 325 326 /* Simple message to indicate that the bootops pointer has been zeroed */ 327 #ifdef DEBUG 328 static int bootops_gone_on = 0; 329 #define BOOTOPS_GONE() \ 330 if (bootops_gone_on) \ 331 prom_printf("The bootops vec is zeroed now!\n"); 332 #else 333 #define BOOTOPS_GONE() 334 #endif /* DEBUG */ 335 336 /* 337 * Monitor pages may not be where this says they are. 338 * and the debugger may not be there either. 339 * 340 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 341 * 342 * Physical memory layout 343 * (not necessarily contiguous) 344 * (THIS IS SOMEWHAT WRONG) 345 * /-----------------------\ 346 * | monitor pages | 347 * availmem -|-----------------------| 348 * | | 349 * | page pool | 350 * | | 351 * |-----------------------| 352 * | configured tables | 353 * | buffers | 354 * firstaddr -|-----------------------| 355 * | hat data structures | 356 * |-----------------------| 357 * | kernel data, bss | 358 * |-----------------------| 359 * | interrupt stack | 360 * |-----------------------| 361 * | kernel text (RO) | 362 * |-----------------------| 363 * | trap table (4k) | 364 * |-----------------------| 365 * page 1 | panicbuf | 366 * |-----------------------| 367 * page 0 | reclaimed | 368 * |_______________________| 369 * 370 * 371 * 372 * Kernel's Virtual Memory Layout. 373 * /-----------------------\ 374 * 0xFFFFFFFF.FFFFFFFF -| |- 375 * | OBP's virtual page | 376 * | tables | 377 * 0xFFFFFFFC.00000000 -|-----------------------|- 378 * : : 379 * : : 380 * -|-----------------------|- 381 * | segzio | (base and size vary) 382 * 0xFFFFFE00.00000000 -|-----------------------|- 383 * | | Ultrasparc I/II support 384 * | segkpm segment | up to 2TB of physical 385 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 386 * | | 387 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 388 * : : 389 * : : 390 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 391 * | | ^ 392 * | UltraSPARC I/II call | | 393 * | bug requires an extra | | 394 * | 4 GB of space between | | 395 * | hole and used RAM | | 396 * | | | 397 * 0xFFFFF800.00000000 -|-----------------------|- | 398 * | | | 399 * | Virtual Address Hole | UltraSPARC 400 * | on UltraSPARC I/II | I/II * ONLY * 401 * | | | 402 * 0x00000800.00000000 -|-----------------------|- | 403 * | | | 404 * | UltraSPARC I/II call | | 405 * | bug requires an extra | | 406 * | 4 GB of space between | | 407 * | hole and used RAM | | 408 * | | v 409 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 410 * : : ^ 411 * : : | 412 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 413 * | | | 414 * | 64-bit kernel ONLY | | 415 * | | | 416 * | kmem64 segment | | 417 * | | | 418 * | (Relocated extra HME | Approximately 419 * | block allocations, | 1 TB of virtual 420 * | memnode freelists, | address space 421 * | HME hash buckets, | | 422 * | mml_table, kpmp_table,| | 423 * | page_t array and | | 424 * | hashblock pool to | | 425 * | avoid hard-coded | | 426 * | 32-bit vaddr | | 427 * | limitations) | | 428 * | | v 429 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 430 * | | 431 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 432 * | | 433 * 0x00000300.00000000 -|-----------------------|- SYSBASE 434 * : : 435 * : : 436 * -|-----------------------|- 437 * | | 438 * | segmap segment | SEGMAPSIZE (1/8th physmem, 439 * | | 256G MAX) 440 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 441 * : : 442 * : : 443 * -|-----------------------|- 444 * | | 445 * | segkp | SEGKPSIZE (2GB) 446 * | | 447 * | | 448 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 449 * | | 450 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 451 * | | (SEGKPBASE - 0x400000) 452 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 453 * | | (MEMSCRUBBASE - NCARGS) 454 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 455 * | | (ARGSBASE - PPMAPSIZE) 456 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 457 * | | 458 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 459 * | | 460 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 461 * : : 462 * : : 463 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 464 * | | 465 * | OBP | 466 * | | 467 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 468 * | kmdb | 469 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 470 * : : 471 * : : 472 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 473 * | | 474 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 475 * | | ~64MB) 476 * 0x00000000.78002000 -|-----------------------| 477 * | panicbuf | 478 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 479 * : : 480 * : : 481 * | | 482 * |-----------------------|- econtig32 483 * | vm structures | 484 * 0x00000000.01C00000 |-----------------------|- nalloc_end 485 * | TSBs | 486 * |-----------------------|- end/nalloc_base 487 * | kernel data & bss | 488 * 0x00000000.01800000 -|-----------------------| 489 * : nucleus text hole : 490 * 0x00000000.01400000 -|-----------------------| 491 * : : 492 * |-----------------------| 493 * | module text | 494 * |-----------------------|- e_text/modtext 495 * | kernel text | 496 * |-----------------------| 497 * | trap table (48k) | 498 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 499 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 500 * |-----------------------| 501 * | | 502 * | invalid | 503 * | | 504 * 0x00000000.00000000 _|_______________________| 505 * 506 * 507 * 508 * 32-bit User Virtual Memory Layout. 509 * /-----------------------\ 510 * | | 511 * | invalid | 512 * | | 513 * 0xFFC00000 -|-----------------------|- USERLIMIT 514 * | user stack | 515 * : : 516 * : : 517 * : : 518 * | user data | 519 * -|-----------------------|- 520 * | user text | 521 * 0x00002000 -|-----------------------|- 522 * | invalid | 523 * 0x00000000 _|_______________________| 524 * 525 * 526 * 527 * 64-bit User Virtual Memory Layout. 528 * /-----------------------\ 529 * | | 530 * | invalid | 531 * | | 532 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 533 * | user stack | 534 * : : 535 * : : 536 * : : 537 * | user data | 538 * -|-----------------------|- 539 * | user text | 540 * 0x00000000.00100000 -|-----------------------|- 541 * | invalid | 542 * 0x00000000.00000000 _|_______________________| 543 */ 544 545 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 546 extern uint64_t ecache_flush_address(void); 547 548 #pragma weak load_platform_modules 549 #pragma weak plat_startup_memlist 550 #pragma weak ecache_init_scrub_flush_area 551 #pragma weak ecache_flush_address 552 553 554 /* 555 * By default the DR Cage is enabled for maximum OS 556 * MPSS performance. Users needing to disable the cage mechanism 557 * can set this variable to zero via /etc/system. 558 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 559 * will result in loss of DR functionality. 560 * Platforms wishing to disable kernel Cage by default 561 * should do so in their set_platform_defaults() routine. 562 */ 563 int kernel_cage_enable = 1; 564 565 static void 566 setup_cage_params(void) 567 { 568 void (*func)(void); 569 570 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 571 if (func != NULL) { 572 (*func)(); 573 return; 574 } 575 576 if (kernel_cage_enable == 0) { 577 return; 578 } 579 kcage_range_lock(); 580 if (kcage_range_init(phys_avail, 1) == 0) { 581 kcage_init(total_pages / 256); 582 } 583 kcage_range_unlock(); 584 585 if (kcage_on) { 586 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 587 } else { 588 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 589 } 590 591 } 592 593 /* 594 * Machine-dependent startup code 595 */ 596 void 597 startup(void) 598 { 599 startup_init(); 600 if (&startup_platform) 601 startup_platform(); 602 startup_memlist(); 603 startup_modules(); 604 setup_cage_params(); 605 startup_bop_gone(); 606 startup_vm(); 607 startup_end(); 608 } 609 610 struct regs sync_reg_buf; 611 uint64_t sync_tt; 612 613 void 614 sync_handler(void) 615 { 616 struct trap_info ti; 617 int i; 618 619 /* 620 * Prevent trying to talk to the other CPUs since they are 621 * sitting in the prom and won't reply. 622 */ 623 for (i = 0; i < NCPU; i++) { 624 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 625 cpu[i]->cpu_flags &= ~CPU_READY; 626 cpu[i]->cpu_flags |= CPU_QUIESCED; 627 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 628 } 629 } 630 631 /* 632 * We've managed to get here without going through the 633 * normal panic code path. Try and save some useful 634 * information. 635 */ 636 if (!panicstr && (curthread->t_panic_trap == NULL)) { 637 ti.trap_type = sync_tt; 638 ti.trap_regs = &sync_reg_buf; 639 ti.trap_addr = NULL; 640 ti.trap_mmu_fsr = 0x0; 641 642 curthread->t_panic_trap = &ti; 643 } 644 645 /* 646 * If we're re-entering the panic path, update the signature 647 * block so that the SC knows we're in the second part of panic. 648 */ 649 if (panicstr) 650 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 651 652 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 653 panic("sync initiated"); 654 } 655 656 657 static void 658 startup_init(void) 659 { 660 /* 661 * We want to save the registers while we're still in OBP 662 * so that we know they haven't been fiddled with since. 663 * (In principle, OBP can't change them just because it 664 * makes a callback, but we'd rather not depend on that 665 * behavior.) 666 */ 667 char sync_str[] = 668 "warning @ warning off : sync " 669 "%%tl-c %%tstate h# %p x! " 670 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 671 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 672 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 673 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 674 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 675 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 676 "%%y h# %p l! %%tl-c %%tt h# %p x! " 677 "sync ; warning !"; 678 679 /* 680 * 20 == num of %p substrings 681 * 16 == max num of chars %p will expand to. 682 */ 683 char bp[sizeof (sync_str) + 16 * 20]; 684 685 (void) check_boot_version(BOP_GETVERSION(bootops)); 686 687 /* 688 * Initialize ptl1 stack for the 1st CPU. 689 */ 690 ptl1_init_cpu(&cpu0); 691 692 /* 693 * Initialize the address map for cache consistent mappings 694 * to random pages; must be done after vac_size is set. 695 */ 696 ppmapinit(); 697 698 /* 699 * Initialize the PROM callback handler. 700 */ 701 init_vx_handler(); 702 703 /* 704 * have prom call sync_callback() to handle the sync and 705 * save some useful information which will be stored in the 706 * core file later. 707 */ 708 (void) sprintf((char *)bp, sync_str, 709 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 710 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 711 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 712 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 713 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 714 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 715 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 716 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 717 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 718 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 719 prom_interpret(bp, 0, 0, 0, 0, 0); 720 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 721 } 722 723 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 724 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 725 726 #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 727 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 728 (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 729 730 /* 731 * As OBP takes up some RAM when the system boots, pages will already be "lost" 732 * to the system and reflected in npages by the time we see it. 733 * 734 * We only want to allocate kernel structures in the 64-bit virtual address 735 * space on systems with enough RAM to make the overhead of keeping track of 736 * an extra kernel memory segment worthwhile. 737 * 738 * Since OBP has already performed its memory allocations by this point, if we 739 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 740 * memory in the 64-bit virtual address space; otherwise keep allocations 741 * contiguous with we've mapped so far in the 32-bit virtual address space. 742 */ 743 #define MINMOVE_RAM_MB ((size_t)1900) 744 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 745 746 pgcnt_t tune_npages = (pgcnt_t) 747 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 748 749 static void 750 startup_memlist(void) 751 { 752 size_t alloc_sz; 753 size_t ctrs_sz; 754 caddr_t alloc_base; 755 caddr_t ctrs_base, ctrs_end; 756 caddr_t memspace; 757 caddr_t va; 758 int memblocks = 0; 759 struct memlist *cur; 760 size_t syslimit = (size_t)SYSLIMIT; 761 size_t sysbase = (size_t)SYSBASE; 762 int alloc_alignsize = MMU_PAGESIZE; 763 extern void page_coloring_init(void); 764 765 /* 766 * Initialize enough of the system to allow kmem_alloc to work by 767 * calling boot to allocate its memory until the time that 768 * kvm_init is completed. The page structs are allocated after 769 * rounding up end to the nearest page boundary; the memsegs are 770 * initialized and the space they use comes from the kernel heap. 771 * With appropriate initialization, they can be reallocated later 772 * to a size appropriate for the machine's configuration. 773 * 774 * At this point, memory is allocated for things that will never 775 * need to be freed, this used to be "valloced". This allows a 776 * savings as the pages don't need page structures to describe 777 * them because them will not be managed by the vm system. 778 */ 779 780 /* 781 * We're loaded by boot with the following configuration (as 782 * specified in the sun4u/conf/Mapfile): 783 * 784 * text: 4 MB chunk aligned on a 4MB boundary 785 * data & bss: 4 MB chunk aligned on a 4MB boundary 786 * 787 * These two chunks will eventually be mapped by 2 locked 4MB 788 * ttes and will represent the nucleus of the kernel. This gives 789 * us some free space that is already allocated, some or all of 790 * which is made available to kernel module text. 791 * 792 * The free space in the data-bss chunk is used for nucleus 793 * allocatable data structures and we reserve it using the 794 * nalloc_base and nalloc_end variables. This space is currently 795 * being used for hat data structures required for tlb miss 796 * handling operations. We align nalloc_base to a l2 cache 797 * linesize because this is the line size the hardware uses to 798 * maintain cache coherency. 799 * 256K is carved out for module data. 800 */ 801 802 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 803 moddata = nalloc_base; 804 e_moddata = nalloc_base + MODDATA; 805 nalloc_base = e_moddata; 806 807 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 808 valloc_base = nalloc_base; 809 810 /* 811 * Calculate the start of the data segment. 812 */ 813 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 814 815 PRM_DEBUG(moddata); 816 PRM_DEBUG(nalloc_base); 817 PRM_DEBUG(nalloc_end); 818 PRM_DEBUG(sdata); 819 820 /* 821 * Remember any slop after e_text so we can give it to the modules. 822 */ 823 PRM_DEBUG(e_text); 824 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 825 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 826 panic("nucleus text overflow"); 827 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 828 modtext; 829 PRM_DEBUG(modtext); 830 PRM_DEBUG(modtext_sz); 831 832 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 833 &boot_physavail, &boot_physavail_len, 834 &boot_virtavail, &boot_virtavail_len); 835 /* 836 * Remember what the physically available highest page is 837 * so that dumpsys works properly, and find out how much 838 * memory is installed. 839 */ 840 installed_top_size_memlist_array(boot_physinstalled, 841 boot_physinstalled_len, &physmax, &physinstalled); 842 PRM_DEBUG(physinstalled); 843 PRM_DEBUG(physmax); 844 845 /* Fill out memory nodes config structure */ 846 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 847 848 /* 849 * Get the list of physically available memory to size 850 * the number of page structures needed. 851 */ 852 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 853 /* 854 * This first snap shot of npages can represent the pages used 855 * by OBP's text and data approximately. This is used in the 856 * the calculation of the kernel size 857 */ 858 obp_pages = physinstalled - npages; 859 860 861 /* 862 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 863 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 864 * 2MB) and any excess pages are put on physavail. The assumption is 865 * that small-memory systems will need more pages more than they'll 866 * need efficiently-mapped module texts. 867 */ 868 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 869 modtext_sz > MODTEXT_SM_CAP) { 870 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 871 modtext_sz = MODTEXT_SM_CAP; 872 } else 873 extra_etpg = 0; 874 PRM_DEBUG(extra_etpg); 875 PRM_DEBUG(modtext_sz); 876 extra_etva = modtext + modtext_sz; 877 PRM_DEBUG(extra_etva); 878 879 /* 880 * Account for any pages after e_text and e_data. 881 */ 882 npages += extra_etpg; 883 npages += mmu_btopr(nalloc_end - nalloc_base); 884 PRM_DEBUG(npages); 885 886 /* 887 * npages is the maximum of available physical memory possible. 888 * (ie. it will never be more than this) 889 */ 890 891 /* 892 * initialize the nucleus memory allocator. 893 */ 894 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 895 896 /* 897 * Allocate mmu fault status area from the nucleus data area. 898 */ 899 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 900 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 901 902 /* 903 * Allocate kernel TSBs from the nucleus data area. 904 */ 905 if (ndata_alloc_tsbs(&ndata, npages) != 0) 906 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 907 908 /* 909 * Allocate dmv dispatch table from the nucleus data area. 910 */ 911 if (ndata_alloc_dmv(&ndata) != 0) 912 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 913 914 915 page_coloring_init(); 916 917 /* 918 * Allocate page_freelists bin headers for memnode 0 from the 919 * nucleus data area. 920 */ 921 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 922 cmn_err(CE_PANIC, 923 "no more nucleus memory after page free lists alloc"); 924 925 if (kpm_enable) { 926 kpm_init(); 927 /* 928 * kpm page space -- Update kpm_npages and make the 929 * same assumption about fragmenting as it is done 930 * for memseg_sz. 931 */ 932 kpm_npages_setup(memblocks + 4); 933 } 934 935 /* 936 * Allocate hat related structs from the nucleus data area. 937 */ 938 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 939 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 940 941 /* 942 * We want to do the BOP_ALLOCs before the real allocation of page 943 * structs in order to not have to allocate page structs for this 944 * memory. We need to calculate a virtual address because we want 945 * the page structs to come before other allocations in virtual address 946 * space. This is so some (if not all) of page structs can actually 947 * live in the nucleus. 948 */ 949 950 /* 951 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 952 * 953 * There are comments all over the SFMMU code warning of dire 954 * consequences if the TSBs are moved out of 32-bit space. This 955 * is largely because the asm code uses "sethi %hi(addr)"-type 956 * instructions which will not provide the expected result if the 957 * address is a 64-bit one. 958 * 959 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 960 */ 961 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 962 alloc_base = sfmmu_ktsb_alloc(alloc_base); 963 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 964 PRM_DEBUG(alloc_base); 965 966 /* 967 * Allocate IOMMU TSB array. We do this here so that the physical 968 * memory gets deducted from the PROM's physical memory list. 969 */ 970 alloc_base = iommu_tsb_init(alloc_base); 971 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 972 ecache_alignsize); 973 PRM_DEBUG(alloc_base); 974 975 /* 976 * Platforms like Starcat and OPL need special structures assigned in 977 * 32-bit virtual address space because their probing routines execute 978 * FCode, and FCode can't handle 64-bit virtual addresses... 979 */ 980 if (&plat_startup_memlist) { 981 alloc_base = plat_startup_memlist(alloc_base); 982 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 983 ecache_alignsize); 984 PRM_DEBUG(alloc_base); 985 } 986 987 /* 988 * If we have enough memory, use 4M pages for alignment because it 989 * greatly reduces the number of TLB misses we take albeit at the cost 990 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 991 * allocation.) Still, the speedup on large memory systems (e.g. > 64 992 * GB) is quite noticeable, so it is worth the effort to do if we can. 993 * 994 * Note, however, that this speedup will only occur if the boot PROM 995 * uses the largest possible MMU page size possible to map memory 996 * requests that are properly aligned and sized (for example, a request 997 * for a multiple of 4MB of memory aligned to a 4MB boundary will 998 * result in a mapping using a 4MB MMU page.) 999 * 1000 * Even then, the large page mappings will only speed things up until 1001 * the startup process proceeds a bit further, as when 1002 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 1003 * kernel it remaps everything but the TSBs using 8K pages anyway... 1004 * 1005 * At some point in the future, sfmmu_map_prom_mappings() will be 1006 * rewritten to copy memory mappings to the kernel using the same MMU 1007 * page sizes the PROM used. When that occurs, if the PROM did use 1008 * large MMU pages to map memory, the alignment/sizing work we're 1009 * doing now should give us a nice extra performance boost, albeit at 1010 * the cost of greater RAM usage... 1011 */ 1012 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1013 MMU_PAGESIZE); 1014 1015 PRM_DEBUG(tune_npages); 1016 PRM_DEBUG(alloc_alignsize); 1017 1018 /* 1019 * Save off where the contiguous allocations to date have ended 1020 * in econtig32. 1021 */ 1022 econtig32 = alloc_base; 1023 PRM_DEBUG(econtig32); 1024 1025 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1026 cmn_err(CE_PANIC, "econtig32 too big"); 1027 1028 /* 1029 * To avoid memory allocation collisions in the 32-bit virtual address 1030 * space, make allocations from this point forward in 64-bit virtual 1031 * address space starting at syslimit and working up. Also use the 1032 * alignment specified by alloc_alignsize, as we may be able to save 1033 * ourselves TLB misses by using larger page sizes if they're 1034 * available. 1035 * 1036 * All this is needed because on large memory systems, the default 1037 * Solaris allocations will collide with SYSBASE32, which is hard 1038 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1039 * kernels, move the allocations to a location in the 64-bit virtual 1040 * address space space, allowing those structures to grow without 1041 * worry. 1042 * 1043 * On current CPUs we'll run out of physical memory address bits before 1044 * we need to worry about the allocations running into anything else in 1045 * VM or the virtual address holes on US-I and II, as there's currently 1046 * about 1 TB of addressable space before the US-I/II VA hole. 1047 */ 1048 kmem64_base = (caddr_t)syslimit; 1049 PRM_DEBUG(kmem64_base); 1050 1051 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1052 1053 /* 1054 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1055 * them here. 1056 */ 1057 if (khme_hash == NULL || uhme_hash == NULL) { 1058 /* 1059 * alloc_hme_buckets() will align alloc_base properly before 1060 * assigning the hash buckets, so we don't need to do it 1061 * before the call... 1062 */ 1063 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1064 1065 PRM_DEBUG(alloc_base); 1066 PRM_DEBUG(khme_hash); 1067 PRM_DEBUG(uhme_hash); 1068 } 1069 1070 /* 1071 * Allocate the remaining page freelists. NUMA systems can 1072 * have lots of page freelists, one per node, which quickly 1073 * outgrow the amount of nucleus memory available. 1074 */ 1075 if (max_mem_nodes > 1) { 1076 int mnode; 1077 caddr_t alloc_start = alloc_base; 1078 1079 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1080 alloc_base = alloc_page_freelists(mnode, alloc_base, 1081 ecache_alignsize); 1082 } 1083 1084 if (alloc_base > alloc_start) { 1085 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1086 alloc_alignsize); 1087 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1088 alloc_base - alloc_start, 1089 alloc_alignsize) != alloc_start) 1090 cmn_err(CE_PANIC, 1091 "Unable to alloc page freelists\n"); 1092 } 1093 1094 PRM_DEBUG(alloc_base); 1095 } 1096 1097 if (!mml_table) { 1098 size_t mmltable_sz; 1099 1100 /* 1101 * We need to allocate the mml_table here because there 1102 * was not enough space within the nucleus. 1103 */ 1104 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1105 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1106 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1107 alloc_alignsize); 1108 1109 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1110 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1111 panic("mml_table alloc failure"); 1112 1113 alloc_base += alloc_sz; 1114 PRM_DEBUG(mml_table); 1115 PRM_DEBUG(alloc_base); 1116 } 1117 1118 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1119 size_t kpmptable_sz; 1120 caddr_t table; 1121 1122 /* 1123 * We need to allocate either kpmp_table or kpmp_stable here 1124 * because there was not enough space within the nucleus. 1125 */ 1126 kpmptable_sz = (kpm_smallpages == 0) ? 1127 sizeof (kpm_hlk_t) * kpmp_table_sz : 1128 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1129 1130 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1131 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1132 alloc_alignsize); 1133 1134 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1135 alloc_alignsize); 1136 1137 if (table != alloc_base) 1138 panic("kpmp_table or kpmp_stable alloc failure"); 1139 1140 if (kpm_smallpages == 0) { 1141 kpmp_table = (kpm_hlk_t *)table; 1142 PRM_DEBUG(kpmp_table); 1143 } else { 1144 kpmp_stable = (kpm_shlk_t *)table; 1145 PRM_DEBUG(kpmp_stable); 1146 } 1147 1148 alloc_base += alloc_sz; 1149 PRM_DEBUG(alloc_base); 1150 } 1151 1152 if (&ecache_init_scrub_flush_area) { 1153 /* 1154 * Pass alloc_base directly, as the routine itself is 1155 * responsible for any special alignment requirements... 1156 */ 1157 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1158 PRM_DEBUG(alloc_base); 1159 } 1160 1161 /* 1162 * Take the most current snapshot we can by calling mem-update. 1163 */ 1164 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1165 &boot_physavail, &boot_physavail_len, 1166 &boot_virtavail, &boot_virtavail_len); 1167 1168 /* 1169 * Reset npages and memblocks based on boot_physavail list. 1170 */ 1171 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1172 PRM_DEBUG(npages); 1173 1174 /* 1175 * Account for extra memory after e_text. 1176 */ 1177 npages += extra_etpg; 1178 1179 /* 1180 * Calculate the largest free memory chunk in the nucleus data area. 1181 * We need to figure out if page structs can fit in there or not. 1182 * We also make sure enough page structs get created for any physical 1183 * memory we might be returning to the system. 1184 */ 1185 ndata_remain_sz = ndata_maxsize(&ndata); 1186 PRM_DEBUG(ndata_remain_sz); 1187 1188 pp_sz = sizeof (struct page) * npages; 1189 1190 /* 1191 * Here's a nice bit of code based on somewhat recursive logic: 1192 * 1193 * If the page array would fit within the nucleus, we want to 1194 * add npages to cover any extra memory we may be returning back 1195 * to the system. 1196 * 1197 * HOWEVER, the page array is sized by calculating the size of 1198 * (struct page * npages), as are the pagehash table, ctrs and 1199 * memseg_list, so the very act of performing the calculation below may 1200 * in fact make the array large enough that it no longer fits in the 1201 * nucleus, meaning there would now be a much larger area of the 1202 * nucleus free that should really be added to npages, which would 1203 * make the page array that much larger, and so on. 1204 * 1205 * This also ignores the memory possibly used in the nucleus for the 1206 * the page hash, ctrs and memseg list and the fact that whether they 1207 * fit there or not varies with the npages calculation below, but we 1208 * don't even factor them into the equation at this point; perhaps we 1209 * should or perhaps we should just take the approach that the few 1210 * extra pages we could add via this calculation REALLY aren't worth 1211 * the hassle... 1212 */ 1213 if (ndata_remain_sz > pp_sz) { 1214 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1215 1216 npages += mmu_btop(spare); 1217 1218 pp_sz = npages * sizeof (struct page); 1219 1220 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1221 } 1222 1223 /* 1224 * If physmem is patched to be non-zero, use it instead of 1225 * the monitor value unless physmem is larger than the total 1226 * amount of memory on hand. 1227 */ 1228 if (physmem == 0 || physmem > npages) 1229 physmem = npages; 1230 1231 /* 1232 * If pp_base is NULL that means the routines above have determined 1233 * the page array will not fit in the nucleus; we'll have to 1234 * BOP_ALLOC() ourselves some space for them. 1235 */ 1236 if (pp_base == NULL) { 1237 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1238 alloc_alignsize); 1239 1240 alloc_sz = roundup(pp_sz, alloc_alignsize); 1241 1242 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1243 alloc_base, alloc_sz, alloc_alignsize)) != 1244 (struct page *)alloc_base) 1245 panic("page alloc failure"); 1246 1247 alloc_base += alloc_sz; 1248 } 1249 1250 /* 1251 * The page structure hash table size is a power of 2 1252 * such that the average hash chain length is PAGE_HASHAVELEN. 1253 */ 1254 page_hashsz = npages / PAGE_HASHAVELEN; 1255 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1256 pagehash_sz = sizeof (struct page *) * page_hashsz; 1257 1258 /* 1259 * We want to TRY to fit the page structure hash table, 1260 * the page size free list counters, the memseg list and 1261 * and the kpm page space in the nucleus if possible. 1262 * 1263 * alloc_sz counts how much memory needs to be allocated by 1264 * BOP_ALLOC(). 1265 */ 1266 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1267 1268 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1269 1270 /* 1271 * Size up per page size free list counters. 1272 */ 1273 ctrs_sz = page_ctrs_sz(); 1274 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1275 1276 if (ctrs_base == NULL) 1277 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1278 1279 /* 1280 * The memseg list is for the chunks of physical memory that 1281 * will be managed by the vm system. The number calculated is 1282 * a guess as boot may fragment it more when memory allocations 1283 * are made before kphysm_init(). Currently, there are two 1284 * allocations before then, so we assume each causes fragmen- 1285 * tation, and add a couple more for good measure. 1286 */ 1287 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1288 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1289 1290 if (memseg_base == NULL) 1291 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1292 1293 1294 if (kpm_enable) { 1295 /* 1296 * kpm page space -- Update kpm_npages and make the 1297 * same assumption about fragmenting as it is done 1298 * for memseg_sz above. 1299 */ 1300 kpm_npages_setup(memblocks + 4); 1301 kpm_pp_sz = (kpm_smallpages == 0) ? 1302 kpm_npages * sizeof (kpm_page_t): 1303 kpm_npages * sizeof (kpm_spage_t); 1304 1305 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1306 ecache_alignsize); 1307 1308 if (kpm_pp_base == NULL) 1309 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1310 kpm_pp_sz; 1311 } 1312 1313 if (alloc_sz > 0) { 1314 uintptr_t bop_base; 1315 1316 /* 1317 * We need extra memory allocated through BOP_ALLOC. 1318 */ 1319 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1320 alloc_alignsize); 1321 1322 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1323 1324 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1325 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1326 panic("system page struct alloc failure"); 1327 1328 alloc_base += alloc_sz; 1329 1330 if (page_hash == NULL) { 1331 page_hash = (struct page **)bop_base; 1332 bop_base = roundup(bop_base + pagehash_sz, 1333 ecache_alignsize); 1334 } 1335 1336 if (ctrs_base == NULL) { 1337 ctrs_base = (caddr_t)bop_base; 1338 bop_base = roundup(bop_base + ctrs_sz, 1339 ecache_alignsize); 1340 } 1341 1342 if (memseg_base == NULL) { 1343 memseg_base = (struct memseg *)bop_base; 1344 bop_base = roundup(bop_base + memseg_sz, 1345 ecache_alignsize); 1346 } 1347 1348 if (kpm_enable && kpm_pp_base == NULL) { 1349 kpm_pp_base = (uintptr_t)bop_base; 1350 bop_base = roundup(bop_base + kpm_pp_sz, 1351 ecache_alignsize); 1352 } 1353 1354 ASSERT(bop_base <= (uintptr_t)alloc_base); 1355 } 1356 1357 /* 1358 * Initialize per page size free list counters. 1359 */ 1360 ctrs_end = page_ctrs_alloc(ctrs_base); 1361 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1362 1363 PRM_DEBUG(page_hash); 1364 PRM_DEBUG(memseg_base); 1365 PRM_DEBUG(kpm_pp_base); 1366 PRM_DEBUG(kpm_pp_sz); 1367 PRM_DEBUG(pp_base); 1368 PRM_DEBUG(pp_sz); 1369 PRM_DEBUG(alloc_base); 1370 1371 #ifdef TRAPTRACE 1372 /* 1373 * Allocate trap trace buffer last so as not to affect 1374 * the 4M alignments of the allocations above on V9 SPARCs... 1375 */ 1376 alloc_base = trap_trace_alloc(alloc_base); 1377 PRM_DEBUG(alloc_base); 1378 #endif /* TRAPTRACE */ 1379 1380 if (kmem64_base) { 1381 /* 1382 * Set the end of the kmem64 segment for V9 SPARCs, if 1383 * appropriate... 1384 */ 1385 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1386 alloc_alignsize); 1387 1388 PRM_DEBUG(kmem64_base); 1389 PRM_DEBUG(kmem64_end); 1390 } 1391 1392 /* 1393 * Allocate space for the interrupt vector table and also for the 1394 * reserved interrupt vector data structures. 1395 */ 1396 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 1397 IVSIZE, MMU_PAGESIZE); 1398 if (memspace != (caddr_t)intr_vec_table) 1399 panic("interrupt vector table allocation failure"); 1400 1401 /* 1402 * The memory lists from boot are allocated from the heap arena 1403 * so that later they can be freed and/or reallocated. 1404 */ 1405 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1406 panic("could not retrieve property \"extent\""); 1407 1408 /* 1409 * Between now and when we finish copying in the memory lists, 1410 * allocations happen so the space gets fragmented and the 1411 * lists longer. Leave enough space for lists twice as long 1412 * as what boot says it has now; roundup to a pagesize. 1413 * Also add space for the final phys-avail copy in the fixup 1414 * routine. 1415 */ 1416 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1417 roundup(IVSIZE, MMU_PAGESIZE)); 1418 memlist_sz *= 4; 1419 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1420 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1421 if (memspace == NULL) 1422 halt("Boot allocation failed."); 1423 1424 memlist = (struct memlist *)memspace; 1425 memlist_end = (char *)memspace + memlist_sz; 1426 1427 PRM_DEBUG(memlist); 1428 PRM_DEBUG(memlist_end); 1429 PRM_DEBUG(sysbase); 1430 PRM_DEBUG(syslimit); 1431 1432 kernelheap_init((void *)sysbase, (void *)syslimit, 1433 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1434 1435 /* 1436 * Take the most current snapshot we can by calling mem-update. 1437 */ 1438 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1439 &boot_physavail, &boot_physavail_len, 1440 &boot_virtavail, &boot_virtavail_len); 1441 1442 /* 1443 * Remove the space used by BOP_ALLOC from the kernel heap 1444 * plus the area actually used by the OBP (if any) 1445 * ignoring virtual addresses in virt_avail, above syslimit. 1446 */ 1447 virt_avail = memlist; 1448 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1449 1450 for (cur = virt_avail; cur->next; cur = cur->next) { 1451 uint64_t range_base, range_size; 1452 1453 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1454 continue; 1455 if (range_base >= (uint64_t)syslimit) 1456 break; 1457 /* 1458 * Limit the range to end at syslimit. 1459 */ 1460 range_size = MIN(cur->next->address, 1461 (uint64_t)syslimit) - range_base; 1462 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1463 0, 0, (void *)range_base, (void *)(range_base + range_size), 1464 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1465 } 1466 1467 phys_avail = memlist; 1468 (void) copy_physavail(boot_physavail, boot_physavail_len, 1469 &memlist, 0, 0); 1470 1471 /* 1472 * Add any extra memory after e_text to the phys_avail list, as long 1473 * as there's at least a page to add. 1474 */ 1475 if (extra_etpg) 1476 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1477 &memlist, &phys_avail); 1478 1479 /* 1480 * Add any extra memory after e_data to the phys_avail list as long 1481 * as there's at least a page to add. Usually, there isn't any, 1482 * since extra HME blocks typically get allocated there first before 1483 * using RAM elsewhere. 1484 */ 1485 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1486 nalloc_base = nalloc_end; 1487 ndata_remain_sz = nalloc_end - nalloc_base; 1488 1489 if (ndata_remain_sz >= MMU_PAGESIZE) 1490 memlist_add(va_to_pa(nalloc_base), 1491 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1492 1493 PRM_DEBUG(memlist); 1494 PRM_DEBUG(memlist_sz); 1495 PRM_DEBUG(memspace); 1496 1497 if ((caddr_t)memlist > (memspace + memlist_sz)) 1498 panic("memlist overflow"); 1499 1500 PRM_DEBUG(pp_base); 1501 PRM_DEBUG(memseg_base); 1502 PRM_DEBUG(npages); 1503 1504 /* 1505 * Initialize the page structures from the memory lists. 1506 */ 1507 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1508 1509 availrmem_initial = availrmem = freemem; 1510 PRM_DEBUG(availrmem); 1511 1512 /* 1513 * Some of the locks depend on page_hashsz being set! 1514 * kmem_init() depends on this; so, keep it here. 1515 */ 1516 page_lock_init(); 1517 1518 /* 1519 * Initialize kernel memory allocator. 1520 */ 1521 kmem_init(); 1522 1523 /* 1524 * Initialize bp_mapin(). 1525 */ 1526 bp_init(shm_alignment, HAT_STRICTORDER); 1527 1528 /* 1529 * Reserve space for panicbuf, intr_vec_table and reserved interrupt 1530 * vector data structures from the 32-bit heap. 1531 */ 1532 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1533 panicbuf, panicbuf + PANICBUFSIZE, 1534 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1535 1536 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1537 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1538 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1539 1540 mem_config_init(); 1541 } 1542 1543 static void 1544 startup_modules(void) 1545 { 1546 int proplen, nhblk1, nhblk8; 1547 size_t nhblksz; 1548 pgcnt_t hblk_pages, pages_per_hblk; 1549 size_t hme8blk_sz, hme1blk_sz; 1550 1551 /* 1552 * Log any optional messages from the boot program 1553 */ 1554 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1555 if (proplen > 0) { 1556 char *msg; 1557 size_t len = (size_t)proplen; 1558 1559 msg = kmem_zalloc(len, KM_SLEEP); 1560 (void) BOP_GETPROP(bootops, "boot-message", msg); 1561 cmn_err(CE_CONT, "?%s\n", msg); 1562 kmem_free(msg, len); 1563 } 1564 1565 /* 1566 * Let the platforms have a chance to change default 1567 * values before reading system file. 1568 */ 1569 if (&set_platform_defaults) 1570 set_platform_defaults(); 1571 1572 /* 1573 * Calculate default settings of system parameters based upon 1574 * maxusers, yet allow to be overridden via the /etc/system file. 1575 */ 1576 param_calc(0); 1577 1578 mod_setup(); 1579 1580 /* 1581 * If this is a positron, complain and halt. 1582 */ 1583 if (&iam_positron && iam_positron()) { 1584 cmn_err(CE_WARN, "This hardware platform is not supported" 1585 " by this release of Solaris.\n"); 1586 #ifdef DEBUG 1587 prom_enter_mon(); /* Type 'go' to resume */ 1588 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1589 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1590 1591 #else /* DEBUG */ 1592 halt(0); 1593 #endif /* DEBUG */ 1594 } 1595 1596 /* 1597 * If we are running firmware that isn't 64-bit ready 1598 * then complain and halt. 1599 */ 1600 do_prom_version_check(); 1601 1602 /* 1603 * Initialize system parameters 1604 */ 1605 param_init(); 1606 1607 /* 1608 * maxmem is the amount of physical memory we're playing with. 1609 */ 1610 maxmem = physmem; 1611 1612 /* Set segkp limits. */ 1613 ncbase = (caddr_t)SEGDEBUGBASE; 1614 ncend = (caddr_t)SEGDEBUGBASE; 1615 1616 /* 1617 * Initialize the hat layer. 1618 */ 1619 hat_init(); 1620 1621 /* 1622 * Initialize segment management stuff. 1623 */ 1624 seg_init(); 1625 1626 /* 1627 * Create the va>tte handler, so the prom can understand 1628 * kernel translations. The handler is installed later, just 1629 * as we are about to take over the trap table from the prom. 1630 */ 1631 create_va_to_tte(); 1632 1633 /* 1634 * Load the forthdebugger (optional) 1635 */ 1636 forthdebug_init(); 1637 1638 /* 1639 * Create OBP node for console input callbacks 1640 * if it is needed. 1641 */ 1642 startup_create_io_node(); 1643 1644 if (modloadonly("fs", "specfs") == -1) 1645 halt("Can't load specfs"); 1646 1647 if (modloadonly("fs", "devfs") == -1) 1648 halt("Can't load devfs"); 1649 1650 if (modloadonly("misc", "swapgeneric") == -1) 1651 halt("Can't load swapgeneric"); 1652 1653 (void) modloadonly("sys", "lbl_edition"); 1654 1655 dispinit(); 1656 1657 /* 1658 * Infer meanings to the members of the idprom buffer. 1659 */ 1660 parse_idprom(); 1661 1662 /* Read cluster configuration data. */ 1663 clconf_init(); 1664 1665 setup_ddi(); 1666 1667 /* 1668 * Lets take this opportunity to load the root device. 1669 */ 1670 if (loadrootmodules() != 0) 1671 debug_enter("Can't load the root filesystem"); 1672 1673 /* 1674 * Load tod driver module for the tod part found on this system. 1675 * Recompute the cpu frequency/delays based on tod as tod part 1676 * tends to keep time more accurately. 1677 */ 1678 if (&load_tod_module) 1679 load_tod_module(); 1680 1681 /* 1682 * Allow platforms to load modules which might 1683 * be needed after bootops are gone. 1684 */ 1685 if (&load_platform_modules) 1686 load_platform_modules(); 1687 1688 setcpudelay(); 1689 1690 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1691 &boot_physavail, &boot_physavail_len, 1692 &boot_virtavail, &boot_virtavail_len); 1693 1694 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1695 1696 /* 1697 * Calculation and allocation of hmeblks needed to remap 1698 * the memory allocated by PROM till now: 1699 * 1700 * (1) calculate how much virtual memory has been bop_alloc'ed. 1701 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1702 * (3) calculate number of hme8blk's needed to remap this memory 1703 * (4) calculate amount of memory that's consumed by these hme8blk's 1704 * (5) add memory calculated in steps (2) and (4) above. 1705 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1706 * (7) calculate number of hme8blk's needed to remap this memory 1707 * (8) calculate amount of memory that's consumed by these hme8blk's 1708 * (9) allocate additional hme1blk's to hold large mappings. 1709 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1710 * to remap everything with 4M mappings. 1711 * (10) account for partially used hblk8's due to non-64K aligned 1712 * PROM mapping entries. 1713 * (11) add memory calculated in steps (8), (9), and (10) above. 1714 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1715 * is not ready yet, this gets bop_alloc'ed. 1716 * (13) there will be very few bop_alloc's after this point before 1717 * trap table takes over 1718 */ 1719 1720 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1721 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1722 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1723 1724 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1725 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1726 nhblk8 = bop_alloc_pages / pages_per_hblk; 1727 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1728 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1729 bop_alloc_pages += hblk_pages; 1730 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1731 nhblk8 = bop_alloc_pages / pages_per_hblk; 1732 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1733 if (nhblk1 < hblk1_min) 1734 nhblk1 = hblk1_min; 1735 if (nhblk8 < hblk8_min) 1736 nhblk8 = hblk8_min; 1737 1738 /* 1739 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1740 * boundary, the number of hblk8's needed to map the entries in the 1741 * boot_virtavail list needs to be adjusted to take this into 1742 * consideration. Thus, we need to add additional hblk8's since it 1743 * is possible that an hblk8 will not have all 8 slots used due to 1744 * alignment constraints. Since there were boot_virtavail_len entries 1745 * in that list, we need to add that many hblk8's to the number 1746 * already calculated to make sure we don't underestimate. 1747 */ 1748 nhblk8 += boot_virtavail_len; 1749 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1750 1751 /* Allocate in pagesize chunks */ 1752 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1753 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1754 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1755 } 1756 1757 static void 1758 startup_bop_gone(void) 1759 { 1760 extern int bop_io_quiesced; 1761 1762 /* 1763 * Destroy the MD initialized at startup 1764 * The startup initializes the MD framework 1765 * using prom and BOP alloc free it now. 1766 */ 1767 mach_descrip_startup_fini(); 1768 1769 /* 1770 * Call back into boot and release boots resources. 1771 */ 1772 BOP_QUIESCE_IO(bootops); 1773 bop_io_quiesced = 1; 1774 1775 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1776 &boot_physavail, &boot_physavail_len, 1777 &boot_virtavail, &boot_virtavail_len); 1778 /* 1779 * Copy physinstalled list into kernel space. 1780 */ 1781 phys_install = memlist; 1782 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1783 1784 /* 1785 * setup physically contiguous area twice as large as the ecache. 1786 * this is used while doing displacement flush of ecaches 1787 */ 1788 if (&ecache_flush_address) { 1789 ecache_flushaddr = ecache_flush_address(); 1790 if (ecache_flushaddr == (uint64_t)-1) { 1791 cmn_err(CE_PANIC, 1792 "startup: no memory to set ecache_flushaddr"); 1793 } 1794 } 1795 1796 /* 1797 * Virtual available next. 1798 */ 1799 ASSERT(virt_avail != NULL); 1800 memlist_free_list(virt_avail); 1801 virt_avail = memlist; 1802 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1803 1804 /* 1805 * Last chance to ask our booter questions .. 1806 */ 1807 } 1808 1809 1810 /* 1811 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1812 * allocations have been performed. We can't call it in startup_bop_gone 1813 * since later operations can cause obp to allocate more memory. 1814 */ 1815 void 1816 startup_fixup_physavail(void) 1817 { 1818 struct memlist *cur; 1819 1820 /* 1821 * take the most current snapshot we can by calling mem-update 1822 */ 1823 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1824 &boot_physavail, &boot_physavail_len, 1825 &boot_virtavail, &boot_virtavail_len); 1826 1827 /* 1828 * Copy phys_avail list, again. 1829 * Both the kernel/boot and the prom have been allocating 1830 * from the original list we copied earlier. 1831 */ 1832 cur = memlist; 1833 (void) copy_physavail(boot_physavail, boot_physavail_len, 1834 &memlist, 0, 0); 1835 1836 /* 1837 * Add any extra memory after e_text we added to the phys_avail list 1838 * back to the old list. 1839 */ 1840 if (extra_etpg) 1841 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1842 &memlist, &cur); 1843 if (ndata_remain_sz >= MMU_PAGESIZE) 1844 memlist_add(va_to_pa(nalloc_base), 1845 (uint64_t)ndata_remain_sz, &memlist, &cur); 1846 1847 /* 1848 * There isn't any bounds checking on the memlist area 1849 * so ensure it hasn't overgrown. 1850 */ 1851 if ((caddr_t)memlist > (caddr_t)memlist_end) 1852 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1853 1854 /* 1855 * The kernel removes the pages that were allocated for it from 1856 * the freelist, but we now have to find any -extra- pages that 1857 * the prom has allocated for it's own book-keeping, and remove 1858 * them from the freelist too. sigh. 1859 */ 1860 fix_prom_pages(phys_avail, cur); 1861 1862 ASSERT(phys_avail != NULL); 1863 memlist_free_list(phys_avail); 1864 phys_avail = cur; 1865 1866 /* 1867 * We're done with boot. Just after this point in time, boot 1868 * gets unmapped, so we can no longer rely on its services. 1869 * Zero the bootops to indicate this fact. 1870 */ 1871 bootops = (struct bootops *)NULL; 1872 BOOTOPS_GONE(); 1873 } 1874 1875 static void 1876 startup_vm(void) 1877 { 1878 size_t i; 1879 struct segmap_crargs a; 1880 struct segkpm_crargs b; 1881 1882 uint64_t avmem; 1883 caddr_t va; 1884 pgcnt_t max_phys_segkp; 1885 int mnode; 1886 1887 extern int use_brk_lpg, use_stk_lpg; 1888 1889 /* 1890 * get prom's mappings, create hments for them and switch 1891 * to the kernel context. 1892 */ 1893 hat_kern_setup(); 1894 1895 /* 1896 * Take over trap table 1897 */ 1898 setup_trap_table(); 1899 1900 /* 1901 * Install the va>tte handler, so that the prom can handle 1902 * misses and understand the kernel table layout in case 1903 * we need call into the prom. 1904 */ 1905 install_va_to_tte(); 1906 1907 /* 1908 * Set a flag to indicate that the tba has been taken over. 1909 */ 1910 tba_taken_over = 1; 1911 1912 /* initialize MMU primary context register */ 1913 mmu_init_kcontext(); 1914 1915 /* 1916 * The boot cpu can now take interrupts, x-calls, x-traps 1917 */ 1918 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1919 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1920 1921 /* 1922 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1923 */ 1924 tbr_wr_addr_inited = 1; 1925 1926 /* 1927 * Initialize VM system, and map kernel address space. 1928 */ 1929 kvm_init(); 1930 1931 /* 1932 * XXX4U: previously, we initialized and turned on 1933 * the caches at this point. But of course we have 1934 * nothing to do, as the prom has already done this 1935 * for us -- main memory must be E$able at all times. 1936 */ 1937 1938 /* 1939 * If the following is true, someone has patched 1940 * phsymem to be less than the number of pages that 1941 * the system actually has. Remove pages until system 1942 * memory is limited to the requested amount. Since we 1943 * have allocated page structures for all pages, we 1944 * correct the amount of memory we want to remove 1945 * by the size of the memory used to hold page structures 1946 * for the non-used pages. 1947 */ 1948 if (physmem < npages) { 1949 pgcnt_t diff, off; 1950 struct page *pp; 1951 struct seg kseg; 1952 1953 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1954 1955 off = 0; 1956 diff = npages - physmem; 1957 diff -= mmu_btopr(diff * sizeof (struct page)); 1958 kseg.s_as = &kas; 1959 while (diff--) { 1960 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1961 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1962 &kseg, (caddr_t)off); 1963 if (pp == NULL) 1964 cmn_err(CE_PANIC, "limited physmem too much!"); 1965 page_io_unlock(pp); 1966 page_downgrade(pp); 1967 availrmem--; 1968 off += MMU_PAGESIZE; 1969 } 1970 } 1971 1972 /* 1973 * When printing memory, show the total as physmem less 1974 * that stolen by a debugger. 1975 */ 1976 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1977 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1978 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1979 1980 avmem = (uint64_t)freemem << PAGESHIFT; 1981 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1982 1983 /* 1984 * For small memory systems disable automatic large pages. 1985 */ 1986 if (physmem < privm_lpg_min_physmem) { 1987 use_brk_lpg = 0; 1988 use_stk_lpg = 0; 1989 } 1990 1991 /* 1992 * Perform platform specific freelist processing 1993 */ 1994 if (&plat_freelist_process) { 1995 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1996 if (mem_node_config[mnode].exists) 1997 plat_freelist_process(mnode); 1998 } 1999 2000 /* 2001 * Initialize the segkp segment type. We position it 2002 * after the configured tables and buffers (whose end 2003 * is given by econtig) and before V_WKBASE_ADDR. 2004 * Also in this area is segkmap (size SEGMAPSIZE). 2005 */ 2006 2007 /* XXX - cache alignment? */ 2008 va = (caddr_t)SEGKPBASE; 2009 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2010 2011 max_phys_segkp = (physmem * 2); 2012 2013 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2014 segkpsize = btop(SEGKPDEFSIZE); 2015 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2016 "segkpsize has been reset to %ld pages", segkpsize); 2017 } 2018 2019 i = ptob(MIN(segkpsize, max_phys_segkp)); 2020 2021 rw_enter(&kas.a_lock, RW_WRITER); 2022 if (seg_attach(&kas, va, i, segkp) < 0) 2023 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2024 if (segkp_create(segkp) != 0) 2025 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2026 rw_exit(&kas.a_lock); 2027 2028 /* 2029 * kpm segment 2030 */ 2031 segmap_kpm = kpm_enable && 2032 segmap_kpm && PAGESIZE == MAXBSIZE; 2033 2034 if (kpm_enable) { 2035 rw_enter(&kas.a_lock, RW_WRITER); 2036 2037 /* 2038 * The segkpm virtual range range is larger than the 2039 * actual physical memory size and also covers gaps in 2040 * the physical address range for the following reasons: 2041 * . keep conversion between segkpm and physical addresses 2042 * simple, cheap and unambiguous. 2043 * . avoid extension/shrink of the the segkpm in case of DR. 2044 * . avoid complexity for handling of virtual addressed 2045 * caches, segkpm and the regular mapping scheme must be 2046 * kept in sync wrt. the virtual color of mapped pages. 2047 * Any accesses to virtual segkpm ranges not backed by 2048 * physical memory will fall through the memseg pfn hash 2049 * and will be handled in segkpm_fault. 2050 * Additional kpm_size spaces needed for vac alias prevention. 2051 */ 2052 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2053 segkpm) < 0) 2054 cmn_err(CE_PANIC, "cannot attach segkpm"); 2055 2056 b.prot = PROT_READ | PROT_WRITE; 2057 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2058 2059 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2060 panic("segkpm_create segkpm"); 2061 2062 rw_exit(&kas.a_lock); 2063 2064 mach_kpm_init(); 2065 } 2066 2067 if (!segzio_fromheap) { 2068 size_t size; 2069 size_t maxsize; 2070 2071 /* size is in bytes, segziosize is in pages */ 2072 if (segziosize == 0) { 2073 size = mmu_ptob(physmem / 2); 2074 } else { 2075 size = mmu_ptob(segziosize); 2076 } 2077 2078 /* maxsize is 3/4th physmem */ 2079 maxsize = mmu_ptob(physmem) - mmu_ptob(physmem / 4); 2080 2081 if (size < SEGZIOMINSIZE) { 2082 size = SEGZIOMINSIZE; 2083 } else if (size > maxsize) { 2084 size = maxsize; 2085 } 2086 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2087 /* put the base of the ZIO segment after the kpm segment */ 2088 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2089 PRM_DEBUG(segziosize); 2090 PRM_DEBUG(segzio_base); 2091 2092 /* 2093 * On some platforms, kvm_init is called after the kpm 2094 * sizes have been determined. On SPARC, kvm_init is called 2095 * before, so we have to attach the kzioseg after kvm is 2096 * initialized, otherwise we'll try to allocate from the boot 2097 * area since the kernel heap hasn't yet been configured. 2098 */ 2099 rw_enter(&kas.a_lock, RW_WRITER); 2100 2101 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2102 &kzioseg); 2103 (void) segkmem_zio_create(&kzioseg); 2104 2105 /* create zio area covering new segment */ 2106 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2107 2108 rw_exit(&kas.a_lock); 2109 } 2110 2111 2112 /* 2113 * Now create generic mapping segment. This mapping 2114 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2115 * virtual address is greater than the amount of free 2116 * memory that is available, then we trim back the 2117 * segment size to that amount 2118 */ 2119 va = (caddr_t)SEGMAPBASE; 2120 2121 /* 2122 * 1201049: segkmap base address must be MAXBSIZE aligned 2123 */ 2124 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2125 2126 /* 2127 * Set size of segmap to percentage of freemem at boot, 2128 * but stay within the allowable range 2129 * Note we take percentage before converting from pages 2130 * to bytes to avoid an overflow on 32-bit kernels. 2131 */ 2132 i = mmu_ptob((freemem * segmap_percent) / 100); 2133 2134 if (i < MINMAPSIZE) 2135 i = MINMAPSIZE; 2136 2137 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2138 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2139 2140 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2141 2142 rw_enter(&kas.a_lock, RW_WRITER); 2143 if (seg_attach(&kas, va, i, segkmap) < 0) 2144 cmn_err(CE_PANIC, "cannot attach segkmap"); 2145 2146 a.prot = PROT_READ | PROT_WRITE; 2147 a.shmsize = shm_alignment; 2148 a.nfreelist = 0; /* use segmap driver defaults */ 2149 2150 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2151 panic("segmap_create segkmap"); 2152 rw_exit(&kas.a_lock); 2153 2154 segdev_init(); 2155 } 2156 2157 static void 2158 startup_end(void) 2159 { 2160 if ((caddr_t)memlist > (caddr_t)memlist_end) 2161 panic("memlist overflow 2"); 2162 memlist_free_block((caddr_t)memlist, 2163 ((caddr_t)memlist_end - (caddr_t)memlist)); 2164 memlist = NULL; 2165 2166 /* enable page_relocation since OBP is now done */ 2167 page_relocate_ready = 1; 2168 2169 /* 2170 * Perform tasks that get done after most of the VM 2171 * initialization has been done but before the clock 2172 * and other devices get started. 2173 */ 2174 kern_setup1(); 2175 2176 /* 2177 * Intialize the VM arenas for allocating physically 2178 * contiguus memory chunk for interrupt queues snd 2179 * allocate/register boot cpu's queues, if any and 2180 * allocate dump buffer for sun4v systems to store 2181 * extra crash information during crash dump 2182 */ 2183 contig_mem_init(); 2184 mach_descrip_init(); 2185 cpu_intrq_setup(CPU); 2186 cpu_intrq_register(CPU); 2187 mach_htraptrace_setup(CPU->cpu_id); 2188 mach_htraptrace_configure(CPU->cpu_id); 2189 mach_dump_buffer_init(); 2190 2191 /* 2192 * Initialize interrupt related stuff 2193 */ 2194 cpu_intr_alloc(CPU, NINTR_THREADS); 2195 2196 (void) splzs(); /* allow hi clock ints but not zs */ 2197 2198 /* 2199 * Initialize errors. 2200 */ 2201 error_init(); 2202 2203 /* 2204 * Note that we may have already used kernel bcopy before this 2205 * point - but if you really care about this, adb the use_hw_* 2206 * variables to 0 before rebooting. 2207 */ 2208 mach_hw_copy_limit(); 2209 2210 /* 2211 * Install the "real" preemption guards before DDI services 2212 * are available. 2213 */ 2214 (void) prom_set_preprom(kern_preprom); 2215 (void) prom_set_postprom(kern_postprom); 2216 CPU->cpu_m.mutex_ready = 1; 2217 2218 /* 2219 * Initialize segnf (kernel support for non-faulting loads). 2220 */ 2221 segnf_init(); 2222 2223 /* 2224 * Configure the root devinfo node. 2225 */ 2226 configure(); /* set up devices */ 2227 mach_cpu_halt_idle(); 2228 } 2229 2230 2231 void 2232 post_startup(void) 2233 { 2234 #ifdef PTL1_PANIC_DEBUG 2235 extern void init_ptl1_thread(void); 2236 #endif /* PTL1_PANIC_DEBUG */ 2237 extern void abort_sequence_init(void); 2238 2239 /* 2240 * Set the system wide, processor-specific flags to be passed 2241 * to userland via the aux vector for performance hints and 2242 * instruction set extensions. 2243 */ 2244 bind_hwcap(); 2245 2246 /* 2247 * Startup memory scrubber (if any) 2248 */ 2249 mach_memscrub(); 2250 2251 /* 2252 * Allocate soft interrupt to handle abort sequence. 2253 */ 2254 abort_sequence_init(); 2255 2256 /* 2257 * Configure the rest of the system. 2258 * Perform forceloading tasks for /etc/system. 2259 */ 2260 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2261 /* 2262 * ON4.0: Force /proc module in until clock interrupt handle fixed 2263 * ON4.0: This must be fixed or restated in /etc/systems. 2264 */ 2265 (void) modload("fs", "procfs"); 2266 2267 /* load machine class specific drivers */ 2268 load_mach_drivers(); 2269 2270 /* load platform specific drivers */ 2271 if (&load_platform_drivers) 2272 load_platform_drivers(); 2273 2274 /* load vis simulation module, if we are running w/fpu off */ 2275 if (!fpu_exists) { 2276 if (modload("misc", "vis") == -1) 2277 halt("Can't load vis"); 2278 } 2279 2280 mach_fpras(); 2281 2282 maxmem = freemem; 2283 2284 #ifdef PTL1_PANIC_DEBUG 2285 init_ptl1_thread(); 2286 #endif /* PTL1_PANIC_DEBUG */ 2287 2288 if (&cif_init) 2289 cif_init(); 2290 } 2291 2292 #ifdef PTL1_PANIC_DEBUG 2293 int ptl1_panic_test = 0; 2294 int ptl1_panic_xc_one_test = 0; 2295 int ptl1_panic_xc_all_test = 0; 2296 int ptl1_panic_xt_one_test = 0; 2297 int ptl1_panic_xt_all_test = 0; 2298 kthread_id_t ptl1_thread_p = NULL; 2299 kcondvar_t ptl1_cv; 2300 kmutex_t ptl1_mutex; 2301 int ptl1_recurse_count_threshold = 0x40; 2302 int ptl1_recurse_trap_threshold = 0x3d; 2303 extern void ptl1_recurse(int, int); 2304 extern void ptl1_panic_xt(int, int); 2305 2306 /* 2307 * Called once per second by timeout() to wake up 2308 * the ptl1_panic thread to see if it should cause 2309 * a trap to the ptl1_panic() code. 2310 */ 2311 /* ARGSUSED */ 2312 static void 2313 ptl1_wakeup(void *arg) 2314 { 2315 mutex_enter(&ptl1_mutex); 2316 cv_signal(&ptl1_cv); 2317 mutex_exit(&ptl1_mutex); 2318 } 2319 2320 /* 2321 * ptl1_panic cross call function: 2322 * Needed because xc_one() and xc_some() can pass 2323 * 64 bit args but ptl1_recurse() expects ints. 2324 */ 2325 static void 2326 ptl1_panic_xc(void) 2327 { 2328 ptl1_recurse(ptl1_recurse_count_threshold, 2329 ptl1_recurse_trap_threshold); 2330 } 2331 2332 /* 2333 * The ptl1 thread waits for a global flag to be set 2334 * and uses the recurse thresholds to set the stack depth 2335 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2336 * or indirectly via the cross call and cross trap functions. 2337 * 2338 * This is useful testing stack overflows and normal 2339 * ptl1_panic() states with a know stack frame. 2340 * 2341 * ptl1_recurse() is an asm function in ptl1_panic.s that 2342 * sets the {In, Local, Out, and Global} registers to a 2343 * know state on the stack and just prior to causing a 2344 * test ptl1_panic trap. 2345 */ 2346 static void 2347 ptl1_thread(void) 2348 { 2349 mutex_enter(&ptl1_mutex); 2350 while (ptl1_thread_p) { 2351 cpuset_t other_cpus; 2352 int cpu_id; 2353 int my_cpu_id; 2354 int target_cpu_id; 2355 int target_found; 2356 2357 if (ptl1_panic_test) { 2358 ptl1_recurse(ptl1_recurse_count_threshold, 2359 ptl1_recurse_trap_threshold); 2360 } 2361 2362 /* 2363 * Find potential targets for x-call and x-trap, 2364 * if any exist while preempt is disabled we 2365 * start a ptl1_panic if requested via a 2366 * globals. 2367 */ 2368 kpreempt_disable(); 2369 my_cpu_id = CPU->cpu_id; 2370 other_cpus = cpu_ready_set; 2371 CPUSET_DEL(other_cpus, CPU->cpu_id); 2372 target_found = 0; 2373 if (!CPUSET_ISNULL(other_cpus)) { 2374 /* 2375 * Pick the first one 2376 */ 2377 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2378 if (cpu_id == my_cpu_id) 2379 continue; 2380 2381 if (CPU_XCALL_READY(cpu_id)) { 2382 target_cpu_id = cpu_id; 2383 target_found = 1; 2384 break; 2385 } 2386 } 2387 ASSERT(target_found); 2388 2389 if (ptl1_panic_xc_one_test) { 2390 xc_one(target_cpu_id, 2391 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2392 } 2393 if (ptl1_panic_xc_all_test) { 2394 xc_some(other_cpus, 2395 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2396 } 2397 if (ptl1_panic_xt_one_test) { 2398 xt_one(target_cpu_id, 2399 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2400 } 2401 if (ptl1_panic_xt_all_test) { 2402 xt_some(other_cpus, 2403 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2404 } 2405 } 2406 kpreempt_enable(); 2407 (void) timeout(ptl1_wakeup, NULL, hz); 2408 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2409 } 2410 mutex_exit(&ptl1_mutex); 2411 } 2412 2413 /* 2414 * Called during early startup to create the ptl1_thread 2415 */ 2416 void 2417 init_ptl1_thread(void) 2418 { 2419 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2420 &p0, TS_RUN, 0); 2421 } 2422 #endif /* PTL1_PANIC_DEBUG */ 2423 2424 2425 /* 2426 * Add to a memory list. 2427 * start = start of new memory segment 2428 * len = length of new memory segment in bytes 2429 * memlistp = pointer to array of available memory segment structures 2430 * curmemlistp = memory list to which to add segment. 2431 */ 2432 static void 2433 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2434 struct memlist **curmemlistp) 2435 { 2436 struct memlist *new; 2437 2438 new = *memlistp; 2439 new->address = start; 2440 new->size = len; 2441 *memlistp = new + 1; 2442 2443 memlist_insert(new, curmemlistp); 2444 } 2445 2446 /* 2447 * In the case of architectures that support dynamic addition of 2448 * memory at run-time there are two cases where memsegs need to 2449 * be initialized and added to the memseg list. 2450 * 1) memsegs that are constructed at startup. 2451 * 2) memsegs that are constructed at run-time on 2452 * hot-plug capable architectures. 2453 * This code was originally part of the function kphysm_init(). 2454 */ 2455 2456 static void 2457 memseg_list_add(struct memseg *memsegp) 2458 { 2459 struct memseg **prev_memsegp; 2460 pgcnt_t num; 2461 2462 /* insert in memseg list, decreasing number of pages order */ 2463 2464 num = MSEG_NPAGES(memsegp); 2465 2466 for (prev_memsegp = &memsegs; *prev_memsegp; 2467 prev_memsegp = &((*prev_memsegp)->next)) { 2468 if (num > MSEG_NPAGES(*prev_memsegp)) 2469 break; 2470 } 2471 2472 memsegp->next = *prev_memsegp; 2473 *prev_memsegp = memsegp; 2474 2475 if (kpm_enable) { 2476 memsegp->nextpa = (memsegp->next) ? 2477 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2478 2479 if (prev_memsegp != &memsegs) { 2480 struct memseg *msp; 2481 msp = (struct memseg *)((caddr_t)prev_memsegp - 2482 offsetof(struct memseg, next)); 2483 msp->nextpa = va_to_pa(memsegp); 2484 } else { 2485 memsegspa = va_to_pa(memsegs); 2486 } 2487 } 2488 } 2489 2490 /* 2491 * PSM add_physmem_cb(). US-II and newer processors have some 2492 * flavor of the prefetch capability implemented. We exploit 2493 * this capability for optimum performance. 2494 */ 2495 #define PREFETCH_BYTES 64 2496 2497 void 2498 add_physmem_cb(page_t *pp, pfn_t pnum) 2499 { 2500 extern void prefetch_page_w(void *); 2501 2502 pp->p_pagenum = pnum; 2503 2504 /* 2505 * Prefetch one more page_t into E$. To prevent future 2506 * mishaps with the sizeof(page_t) changing on us, we 2507 * catch this on debug kernels if we can't bring in the 2508 * entire hpage with 2 PREFETCH_BYTES reads. See 2509 * also, sun4u/cpu/cpu_module.c 2510 */ 2511 /*LINTED*/ 2512 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2513 prefetch_page_w((char *)pp); 2514 } 2515 2516 /* 2517 * kphysm_init() tackles the problem of initializing physical memory. 2518 * The old startup made some assumptions about the kernel living in 2519 * physically contiguous space which is no longer valid. 2520 */ 2521 static void 2522 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2523 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2524 { 2525 struct memlist *pmem; 2526 struct memseg *msp; 2527 pfn_t base; 2528 pgcnt_t num; 2529 pfn_t lastseg_pages_end = 0; 2530 pgcnt_t nelem_used = 0; 2531 2532 ASSERT(page_hash != NULL && page_hashsz != 0); 2533 2534 msp = memsegp; 2535 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2536 2537 /* 2538 * Build the memsegs entry 2539 */ 2540 num = btop(pmem->size); 2541 if (num > npages) 2542 num = npages; 2543 npages -= num; 2544 base = btop(pmem->address); 2545 2546 msp->pages = pp; 2547 msp->epages = pp + num; 2548 msp->pages_base = base; 2549 msp->pages_end = base + num; 2550 2551 if (kpm_enable) { 2552 pfn_t pbase_a; 2553 pfn_t pend_a; 2554 pfn_t prev_pend_a; 2555 pgcnt_t nelem; 2556 2557 msp->pagespa = va_to_pa(pp); 2558 msp->epagespa = va_to_pa(pp + num); 2559 pbase_a = kpmptop(ptokpmp(base)); 2560 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2561 nelem = ptokpmp(pend_a - pbase_a); 2562 msp->kpm_nkpmpgs = nelem; 2563 msp->kpm_pbase = pbase_a; 2564 if (lastseg_pages_end) { 2565 /* 2566 * Assume phys_avail is in ascending order 2567 * of physical addresses. 2568 */ 2569 ASSERT(base + num > lastseg_pages_end); 2570 prev_pend_a = kpmptop( 2571 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2572 2573 if (prev_pend_a > pbase_a) { 2574 /* 2575 * Overlap, more than one memseg may 2576 * point to the same kpm_page range. 2577 */ 2578 if (kpm_smallpages == 0) { 2579 msp->kpm_pages = 2580 (kpm_page_t *)kpm_pp - 1; 2581 kpm_pp = (uintptr_t) 2582 ((kpm_page_t *)kpm_pp 2583 + nelem - 1); 2584 } else { 2585 msp->kpm_spages = 2586 (kpm_spage_t *)kpm_pp - 1; 2587 kpm_pp = (uintptr_t) 2588 ((kpm_spage_t *)kpm_pp 2589 + nelem - 1); 2590 } 2591 nelem_used += nelem - 1; 2592 2593 } else { 2594 if (kpm_smallpages == 0) { 2595 msp->kpm_pages = 2596 (kpm_page_t *)kpm_pp; 2597 kpm_pp = (uintptr_t) 2598 ((kpm_page_t *)kpm_pp 2599 + nelem); 2600 } else { 2601 msp->kpm_spages = 2602 (kpm_spage_t *)kpm_pp; 2603 kpm_pp = (uintptr_t) 2604 ((kpm_spage_t *) 2605 kpm_pp + nelem); 2606 } 2607 nelem_used += nelem; 2608 } 2609 2610 } else { 2611 if (kpm_smallpages == 0) { 2612 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2613 kpm_pp = (uintptr_t) 2614 ((kpm_page_t *)kpm_pp + nelem); 2615 } else { 2616 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2617 kpm_pp = (uintptr_t) 2618 ((kpm_spage_t *)kpm_pp + nelem); 2619 } 2620 nelem_used = nelem; 2621 } 2622 2623 if (nelem_used > kpm_npages) 2624 panic("kphysm_init: kpm_pp overflow\n"); 2625 2626 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2627 lastseg_pages_end = msp->pages_end; 2628 } 2629 2630 memseg_list_add(msp); 2631 2632 /* 2633 * add_physmem() initializes the PSM part of the page 2634 * struct by calling the PSM back with add_physmem_cb(). 2635 * In addition it coalesces pages into larger pages as 2636 * it initializes them. 2637 */ 2638 add_physmem(pp, num, base); 2639 pp += num; 2640 msp++; 2641 } 2642 2643 build_pfn_hash(); 2644 } 2645 2646 /* 2647 * Kernel VM initialization. 2648 * Assumptions about kernel address space ordering: 2649 * (1) gap (user space) 2650 * (2) kernel text 2651 * (3) kernel data/bss 2652 * (4) gap 2653 * (5) kernel data structures 2654 * (6) gap 2655 * (7) debugger (optional) 2656 * (8) monitor 2657 * (9) gap (possibly null) 2658 * (10) dvma 2659 * (11) devices 2660 */ 2661 static void 2662 kvm_init(void) 2663 { 2664 /* 2665 * Put the kernel segments in kernel address space. 2666 */ 2667 rw_enter(&kas.a_lock, RW_WRITER); 2668 as_avlinit(&kas); 2669 2670 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2671 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2672 (void) segkmem_create(&ktextseg); 2673 2674 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2675 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2676 (void) segkmem_create(&ktexthole); 2677 2678 (void) seg_attach(&kas, (caddr_t)valloc_base, 2679 (size_t)(econtig32 - valloc_base), &kvalloc); 2680 (void) segkmem_create(&kvalloc); 2681 2682 if (kmem64_base) { 2683 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2684 (size_t)(kmem64_end - kmem64_base), &kmem64); 2685 (void) segkmem_create(&kmem64); 2686 } 2687 2688 /* 2689 * We're about to map out /boot. This is the beginning of the 2690 * system resource management transition. We can no longer 2691 * call into /boot for I/O or memory allocations. 2692 */ 2693 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2694 (void) segkmem_create(&kvseg); 2695 hblk_alloc_dynamic = 1; 2696 2697 /* 2698 * we need to preallocate pages for DR operations before enabling large 2699 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2700 */ 2701 memseg_remap_init(); 2702 2703 /* at this point we are ready to use large page heap */ 2704 segkmem_heap_lp_init(); 2705 2706 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2707 &kvseg32); 2708 (void) segkmem_create(&kvseg32); 2709 2710 /* 2711 * Create a segment for the debugger. 2712 */ 2713 (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2714 &kdebugseg); 2715 (void) segkmem_create(&kdebugseg); 2716 2717 rw_exit(&kas.a_lock); 2718 } 2719 2720 char obp_tte_str[] = 2721 "h# %x constant MMU_PAGESHIFT " 2722 "h# %x constant TTE8K " 2723 "h# %x constant SFHME_SIZE " 2724 "h# %x constant SFHME_TTE " 2725 "h# %x constant HMEBLK_TAG " 2726 "h# %x constant HMEBLK_NEXT " 2727 "h# %x constant HMEBLK_MISC " 2728 "h# %x constant HMEBLK_HME1 " 2729 "h# %x constant NHMENTS " 2730 "h# %x constant HBLK_SZMASK " 2731 "h# %x constant HBLK_RANGE_SHIFT " 2732 "h# %x constant HMEBP_HBLK " 2733 "h# %x constant HMEBUCKET_SIZE " 2734 "h# %x constant HTAG_SFMMUPSZ " 2735 "h# %x constant HTAG_REHASHSZ " 2736 "h# %x constant mmu_hashcnt " 2737 "h# %p constant uhme_hash " 2738 "h# %p constant khme_hash " 2739 "h# %x constant UHMEHASH_SZ " 2740 "h# %x constant KHMEHASH_SZ " 2741 "h# %p constant KCONTEXT " 2742 "h# %p constant KHATID " 2743 "h# %x constant ASI_MEM " 2744 2745 ": PHYS-X@ ( phys -- data ) " 2746 " ASI_MEM spacex@ " 2747 "; " 2748 2749 ": PHYS-W@ ( phys -- data ) " 2750 " ASI_MEM spacew@ " 2751 "; " 2752 2753 ": PHYS-L@ ( phys -- data ) " 2754 " ASI_MEM spaceL@ " 2755 "; " 2756 2757 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2758 " 3 * MMU_PAGESHIFT + " 2759 "; " 2760 2761 ": TTE_IS_VALID ( ttep -- flag ) " 2762 " PHYS-X@ 0< " 2763 "; " 2764 2765 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2766 " dup TTE8K = if " 2767 " drop HBLK_RANGE_SHIFT " 2768 " else " 2769 " TTE_PAGE_SHIFT " 2770 " then " 2771 "; " 2772 2773 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2774 " tuck >> swap MMU_PAGESHIFT - << " 2775 "; " 2776 2777 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2778 " >> over xor swap ( hash sfmmup ) " 2779 " KHATID <> if ( hash ) " 2780 " UHMEHASH_SZ and ( bucket ) " 2781 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2782 " else ( hash ) " 2783 " KHMEHASH_SZ and ( bucket ) " 2784 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2785 " then ( hmebp ) " 2786 "; " 2787 2788 ": HME_HASH_TABLE_SEARCH " 2789 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2790 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2791 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2792 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2793 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2794 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2795 " else " 2796 " hmeblk_next + phys-x@ false " 2797 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2798 " then " 2799 " else " 2800 " hmeblk_next + phys-x@ false " 2801 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2802 " then " 2803 " else " 2804 " true " 2805 " then " 2806 " until r> drop " 2807 "; " 2808 2809 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2810 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2811 " HTAG_REHASHSZ << or nip ( hblktag ) " 2812 "; " 2813 2814 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2815 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2816 " TTE8K = if ( hmeblkp addr ) " 2817 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2818 " else ( hmeblkp addr ) " 2819 " drop 0 ( hmeblkp 0 ) " 2820 " then ( hmeblkp hme-index ) " 2821 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2822 " SFHME_TTE + ( ttep ) " 2823 "; " 2824 2825 ": unix-tte ( addr cnum -- false | tte-data true ) " 2826 " KCONTEXT = if ( addr ) " 2827 " KHATID ( addr khatid ) " 2828 " else ( addr ) " 2829 " drop false exit ( false ) " 2830 " then " 2831 " ( addr khatid ) " 2832 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2833 " 2dup swap i HME_HASH_SHIFT " 2834 "( addr sfmmup sfmmup addr hmeshift ) " 2835 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2836 " over i 4 pick " 2837 "( addr sfmmup hmebp sfmmup rehash addr ) " 2838 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2839 " HME_HASH_TABLE_SEARCH " 2840 "( addr sfmmup { null | hmeblkp } ) " 2841 " ?dup if ( addr sfmmup hmeblkp ) " 2842 " nip swap HBLK_TO_TTEP ( ttep ) " 2843 " dup TTE_IS_VALID if ( valid-ttep ) " 2844 " PHYS-X@ true ( tte-data true ) " 2845 " else ( invalid-tte ) " 2846 " drop false ( false ) " 2847 " then ( false | tte-data true ) " 2848 " unloop exit ( false | tte-data true ) " 2849 " then ( addr sfmmup ) " 2850 " loop ( addr sfmmup ) " 2851 " 2drop false ( false ) " 2852 "; " 2853 ; 2854 2855 void 2856 create_va_to_tte(void) 2857 { 2858 char *bp; 2859 extern int khmehash_num, uhmehash_num; 2860 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2861 2862 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2863 2864 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2865 2866 /* 2867 * Teach obp how to parse our sw ttes. 2868 */ 2869 (void) sprintf(bp, obp_tte_str, 2870 MMU_PAGESHIFT, 2871 TTE8K, 2872 sizeof (struct sf_hment), 2873 OFFSET(struct sf_hment, hme_tte), 2874 OFFSET(struct hme_blk, hblk_tag), 2875 OFFSET(struct hme_blk, hblk_nextpa), 2876 OFFSET(struct hme_blk, hblk_misc), 2877 OFFSET(struct hme_blk, hblk_hme), 2878 NHMENTS, 2879 HBLK_SZMASK, 2880 HBLK_RANGE_SHIFT, 2881 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2882 sizeof (struct hmehash_bucket), 2883 HTAG_SFMMUPSZ, 2884 HTAG_REHASHSZ, 2885 mmu_hashcnt, 2886 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2887 (caddr_t)va_to_pa((caddr_t)khme_hash), 2888 UHMEHASH_SZ, 2889 KHMEHASH_SZ, 2890 KCONTEXT, 2891 KHATID, 2892 ASI_MEM); 2893 prom_interpret(bp, 0, 0, 0, 0, 0); 2894 2895 kobj_free(bp, MMU_PAGESIZE); 2896 } 2897 2898 void 2899 install_va_to_tte(void) 2900 { 2901 /* 2902 * advise prom that he can use unix-tte 2903 */ 2904 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2905 } 2906 2907 2908 /* 2909 * Because kmdb links prom_stdout_is_framebuffer into its own 2910 * module, we add "device-type=display" here for /os-io node, so that 2911 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 2912 * is registered into OBP. 2913 */ 2914 static char *create_node = 2915 "\" /\" find-device " 2916 "new-device " 2917 "\" os-io\" device-name " 2918 "\" display\" device-type " 2919 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2920 " 2>r swap 2 2r> ['] $callback catch if " 2921 " 2drop 3drop 0 " 2922 " then " 2923 "; " 2924 ": read ( adr,len -- #read ) " 2925 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2926 " ( retN ... ret1 N ) " 2927 " ?dup if " 2928 " swap >r 1- 0 ?do drop loop r> " 2929 " else " 2930 " -2 " 2931 " then " 2932 "; " 2933 ": write ( adr,len -- #written ) " 2934 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2935 " ( retN ... ret1 N ) " 2936 " ?dup if " 2937 " swap >r 1- 0 ?do drop loop r> " 2938 " else " 2939 " 0 " 2940 " then " 2941 "; " 2942 ": poll-tty ( -- ) ; " 2943 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2944 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2945 ": cb-give/take ( $method -- ) " 2946 " 0 -rot ['] $callback catch ?dup if " 2947 " >r 2drop 2drop r> throw " 2948 " else " 2949 " 0 ?do drop loop " 2950 " then " 2951 "; " 2952 ": give ( -- ) \" exit-input\" cb-give/take ; " 2953 ": take ( -- ) \" enter-input\" cb-give/take ; " 2954 ": open ( -- ok? ) true ; " 2955 ": close ( -- ) ; " 2956 "finish-device " 2957 "device-end "; 2958 2959 /* 2960 * Create the OBP input/output node (FCode serial driver). 2961 * It is needed for both USB console keyboard and for 2962 * the kernel terminal emulator. It is too early to check for a 2963 * kernel console compatible framebuffer now, so we create this 2964 * so that we're ready if we need to enable kernel terminal emulation. 2965 * 2966 * When the USB software takes over the input device at the time 2967 * consconfig runs, OBP's stdin is redirected to this node. 2968 * Whenever the FORTH user interface is used after this switch, 2969 * the node will call back into the kernel for console input. 2970 * If a serial device such as ttya or a UART with a Type 5 keyboard 2971 * attached is used, OBP takes over the serial device when the system 2972 * goes to the debugger after the system is booted. This sharing 2973 * of the relatively simple serial device is difficult but possible. 2974 * Sharing the USB host controller is impossible due its complexity. 2975 * 2976 * Similarly to USB keyboard input redirection, after consconfig_dacf 2977 * configures a kernel console framebuffer as the standard output 2978 * device, OBP's stdout is switched to to vector through the 2979 * /os-io node into the kernel terminal emulator. 2980 */ 2981 static void 2982 startup_create_io_node(void) 2983 { 2984 prom_interpret(create_node, 0, 0, 0, 0, 0); 2985 } 2986 2987 2988 static void 2989 do_prom_version_check(void) 2990 { 2991 int i; 2992 pnode_t node; 2993 char buf[64]; 2994 static char drev[] = "Down-rev firmware detected%s\n" 2995 "\tPlease upgrade to the following minimum version:\n" 2996 "\t\t%s\n"; 2997 2998 i = prom_version_check(buf, sizeof (buf), &node); 2999 3000 if (i == PROM_VER64_OK) 3001 return; 3002 3003 if (i == PROM_VER64_UPGRADE) { 3004 cmn_err(CE_WARN, drev, "", buf); 3005 3006 #ifdef DEBUG 3007 prom_enter_mon(); /* Type 'go' to continue */ 3008 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3009 return; 3010 #else 3011 halt(0); 3012 #endif 3013 } 3014 3015 /* 3016 * The other possibility is that this is a server running 3017 * good firmware, but down-rev firmware was detected on at 3018 * least one other cpu board. We just complain if we see 3019 * that. 3020 */ 3021 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3022 } 3023 3024 static void 3025 kpm_init() 3026 { 3027 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 3028 kpm_pgsz = 1ull << kpm_pgshft; 3029 kpm_pgoff = kpm_pgsz - 1; 3030 kpmp2pshft = kpm_pgshft - PAGESHIFT; 3031 kpmpnpgs = 1 << kpmp2pshft; 3032 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 3033 } 3034 3035 void 3036 kpm_npages_setup(int memblocks) 3037 { 3038 /* 3039 * npages can be scattered in a maximum of 'memblocks' 3040 */ 3041 kpm_npages = ptokpmpr(npages) + memblocks; 3042 } 3043 3044 /* 3045 * Must be defined in platform dependent code. 3046 */ 3047 extern caddr_t modtext; 3048 extern size_t modtext_sz; 3049 extern caddr_t moddata; 3050 3051 #define HEAPTEXT_ARENA(addr) \ 3052 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3053 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3054 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3055 3056 #define HEAPTEXT_OVERSIZED(addr) \ 3057 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3058 3059 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3060 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3061 kmutex_t texthole_lock; 3062 3063 char kern_bootargs[OBP_MAXPATHLEN]; 3064 3065 void 3066 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3067 { 3068 uintptr_t addr, limit; 3069 3070 addr = HEAPTEXT_BASE; 3071 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3072 3073 /* 3074 * Before we initialize the text_arena, we want to punch holes in the 3075 * underlying heaptext_arena. This guarantees that for any text 3076 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3077 */ 3078 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3079 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3080 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3081 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3082 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3083 } 3084 3085 /* 3086 * Allocate one page at the oversize to break up the text region 3087 * from the oversized region. 3088 */ 3089 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3090 (void *)limit, (void *)(limit + PAGESIZE), 3091 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3092 3093 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3094 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3095 heaptext_arena, 0, VM_SLEEP); 3096 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3097 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3098 } 3099 3100 caddr_t 3101 kobj_text_alloc(vmem_t *arena, size_t size) 3102 { 3103 caddr_t rval, better; 3104 3105 /* 3106 * First, try a sleeping allocation. 3107 */ 3108 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3109 3110 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3111 return (rval); 3112 3113 /* 3114 * We didn't get the area that we wanted. We're going to try to do an 3115 * allocation with explicit constraints. 3116 */ 3117 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3118 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3119 VM_NOSLEEP | VM_BESTFIT); 3120 3121 if (better != NULL) { 3122 /* 3123 * That worked. Free our first attempt and return. 3124 */ 3125 vmem_free(arena, rval, size); 3126 return (better); 3127 } 3128 3129 /* 3130 * That didn't work; we'll have to return our first attempt. 3131 */ 3132 return (rval); 3133 } 3134 3135 caddr_t 3136 kobj_texthole_alloc(caddr_t addr, size_t size) 3137 { 3138 int arena = HEAPTEXT_ARENA(addr); 3139 char c[30]; 3140 uintptr_t base; 3141 3142 if (HEAPTEXT_OVERSIZED(addr)) { 3143 /* 3144 * If this is an oversized allocation, there is no text hole 3145 * available for it; return NULL. 3146 */ 3147 return (NULL); 3148 } 3149 3150 mutex_enter(&texthole_lock); 3151 3152 if (texthole_arena[arena] == NULL) { 3153 ASSERT(texthole_source[arena] == NULL); 3154 3155 if (arena == 0) { 3156 texthole_source[0] = vmem_create("module_text_holesrc", 3157 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3158 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3159 0, VM_SLEEP); 3160 } else { 3161 base = HEAPTEXT_BASE + 3162 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3163 3164 (void) snprintf(c, sizeof (c), 3165 "heaptext_holesrc_%d", arena); 3166 3167 texthole_source[arena] = vmem_create(c, (void *)base, 3168 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3169 0, VM_SLEEP); 3170 } 3171 3172 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3173 3174 texthole_arena[arena] = vmem_create(c, NULL, 0, 3175 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3176 texthole_source[arena], 0, VM_SLEEP); 3177 } 3178 3179 mutex_exit(&texthole_lock); 3180 3181 ASSERT(texthole_arena[arena] != NULL); 3182 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3183 return (vmem_alloc(texthole_arena[arena], size, 3184 VM_BESTFIT | VM_NOSLEEP)); 3185 } 3186 3187 void 3188 kobj_texthole_free(caddr_t addr, size_t size) 3189 { 3190 int arena = HEAPTEXT_ARENA(addr); 3191 3192 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3193 ASSERT(texthole_arena[arena] != NULL); 3194 vmem_free(texthole_arena[arena], addr, size); 3195 } 3196