1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 65 #ifdef TRAPTRACE 66 #include <sys/traptrace.h> 67 #endif /* TRAPTRACE */ 68 69 #include <sys/memnode.h> 70 #include <sys/mem_cage.h> 71 72 73 extern void setup_trap_table(void); 74 extern void cpu_intrq_setup(struct cpu *); 75 extern void cpu_intrq_register(struct cpu *); 76 extern void contig_mem_init(void); 77 extern void mach_dump_buffer_init(void); 78 extern void mach_descrip_init(void); 79 extern void mach_memscrub(void); 80 extern void mach_fpras(void); 81 extern void mach_cpu_halt_idle(void); 82 extern void mach_hw_copy_limit(void); 83 extern void load_tod_module(void); 84 #pragma weak load_tod_module 85 86 extern int ndata_alloc_mmfsa(struct memlist *ndata); 87 #pragma weak ndata_alloc_mmfsa 88 89 extern void parse_idprom(void); 90 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91 extern void mem_config_init(void); 92 extern void memseg_remap_init(void); 93 94 /* 95 * External Data: 96 */ 97 extern int vac_size; /* cache size in bytes */ 98 extern uint_t vac_mask; /* VAC alignment consistency mask */ 99 extern uint_t vac_colors; 100 101 /* 102 * Global Data Definitions: 103 */ 104 105 /* 106 * XXX - Don't port this to new architectures 107 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 108 * 'romp' has no use with a prom with an IEEE 1275 client interface. 109 * The driver doesn't use the value, but it depends on the symbol. 110 */ 111 void *romp; /* veritas driver won't load without romp 4154976 */ 112 /* 113 * Declare these as initialized data so we can patch them. 114 */ 115 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 116 pgcnt_t segkpsize = 117 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 118 uint_t segmap_percent = 12; /* Size of segmap segment */ 119 120 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 121 int vac_copyback = 1; 122 char *cache_mode = NULL; 123 int use_mix = 1; 124 int prom_debug = 0; 125 int usb_node_debug = 0; 126 127 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 128 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 129 uint_t tba_taken_over = 0; 130 131 caddr_t s_text; /* start of kernel text segment */ 132 caddr_t e_text; /* end of kernel text segment */ 133 caddr_t s_data; /* start of kernel data segment */ 134 caddr_t e_data; /* end of kernel data segment */ 135 136 caddr_t modtext; /* beginning of module text */ 137 size_t modtext_sz; /* size of module text */ 138 caddr_t moddata; /* beginning of module data reserve */ 139 caddr_t e_moddata; /* end of module data reserve */ 140 141 /* 142 * End of first block of contiguous kernel in 32-bit virtual address space 143 */ 144 caddr_t econtig32; /* end of first blk of contiguous kernel */ 145 146 caddr_t ncbase; /* beginning of non-cached segment */ 147 caddr_t ncend; /* end of non-cached segment */ 148 caddr_t sdata; /* beginning of data segment */ 149 150 caddr_t extra_etva; /* beginning of unused nucleus text */ 151 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 152 153 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 154 caddr_t nalloc_base; /* beginning of nucleus allocation */ 155 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 156 caddr_t valloc_base; /* beginning of kvalloc segment */ 157 158 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 159 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 160 161 uintptr_t shm_alignment = 0; /* VAC address consistency modulus */ 162 struct memlist *phys_install; /* Total installed physical memory */ 163 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 164 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 165 struct memlist ndata; /* memlist of nucleus allocatable memory */ 166 int memexp_flag; /* memory expansion card flag */ 167 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 168 pgcnt_t obp_pages; /* Physical pages used by OBP */ 169 170 /* 171 * VM data structures 172 */ 173 long page_hashsz; /* Size of page hash table (power of two) */ 174 struct page *pp_base; /* Base of system page struct array */ 175 size_t pp_sz; /* Size in bytes of page struct array */ 176 struct page **page_hash; /* Page hash table */ 177 struct seg ktextseg; /* Segment used for kernel executable image */ 178 struct seg kvalloc; /* Segment used for "valloc" mapping */ 179 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 180 struct seg ktexthole; /* Segment used for nucleus text hole */ 181 struct seg kmapseg; /* Segment used for generic kernel mappings */ 182 struct seg kpmseg; /* Segment used for physical mapping */ 183 struct seg kdebugseg; /* Segment used for the kernel debugger */ 184 185 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 186 size_t kpm_pp_sz; /* Size of system kpm_page array */ 187 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 188 189 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 190 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 191 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 192 193 /* 194 * debugger pages (if allocated) 195 */ 196 struct vnode kdebugvp; 197 198 /* 199 * Segment for relocated kernel structures in 64-bit large RAM kernels 200 */ 201 struct seg kmem64; 202 203 struct memseg *memseg_base; 204 size_t memseg_sz; /* Used to translate a va to page */ 205 struct vnode unused_pages_vp; 206 207 /* 208 * VM data structures allocated early during boot. 209 */ 210 size_t pagehash_sz; 211 uint64_t memlist_sz; 212 213 char tbr_wr_addr_inited = 0; 214 215 216 /* 217 * Static Routines: 218 */ 219 static void memlist_add(uint64_t, uint64_t, struct memlist **, 220 struct memlist **); 221 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 222 pgcnt_t); 223 static void kvm_init(void); 224 225 static void startup_init(void); 226 static void startup_memlist(void); 227 static void startup_modules(void); 228 static void startup_bop_gone(void); 229 static void startup_vm(void); 230 static void startup_end(void); 231 static void setup_cage_params(void); 232 static void startup_create_input_node(void); 233 234 static pgcnt_t npages; 235 static struct memlist *memlist; 236 void *memlist_end; 237 238 static pgcnt_t bop_alloc_pages; 239 static caddr_t hblk_base; 240 uint_t hblk_alloc_dynamic = 0; 241 uint_t hblk1_min = H1MIN; 242 uint_t hblk8_min; 243 244 245 /* 246 * Hooks for unsupported platforms and down-rev firmware 247 */ 248 int iam_positron(void); 249 #pragma weak iam_positron 250 static void do_prom_version_check(void); 251 static void kpm_init(void); 252 static void kpm_npages_setup(int); 253 static void kpm_memseg_init(void); 254 255 /* 256 * After receiving a thermal interrupt, this is the number of seconds 257 * to delay before shutting off the system, assuming 258 * shutdown fails. Use /etc/system to change the delay if this isn't 259 * large enough. 260 */ 261 int thermal_powerdown_delay = 1200; 262 263 /* 264 * Used to hold off page relocations into the cage until OBP has completed 265 * its boot-time handoff of its resources to the kernel. 266 */ 267 int page_relocate_ready = 0; 268 269 /* 270 * Enable some debugging messages concerning memory usage... 271 */ 272 #ifdef DEBUGGING_MEM 273 static int debugging_mem; 274 static void 275 printmemlist(char *title, struct memlist *list) 276 { 277 if (!debugging_mem) 278 return; 279 280 printf("%s\n", title); 281 282 while (list) { 283 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 284 (uint32_t)(list->address >> 32), (uint32_t)list->address, 285 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 286 list = list->next; 287 } 288 } 289 290 void 291 printmemseg(struct memseg *memseg) 292 { 293 if (!debugging_mem) 294 return; 295 296 printf("memseg\n"); 297 298 while (memseg) { 299 prom_printf("\tpage = 0x%p, epage = 0x%p, " 300 "pfn = 0x%x, epfn = 0x%x\n", 301 memseg->pages, memseg->epages, 302 memseg->pages_base, memseg->pages_end); 303 memseg = memseg->next; 304 } 305 } 306 307 #define debug_pause(str) halt((str)) 308 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 309 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 310 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 311 #define MPRINTF3(str, a, b, c) \ 312 if (debugging_mem) prom_printf((str), (a), (b), (c)) 313 #else /* DEBUGGING_MEM */ 314 #define MPRINTF(str) 315 #define MPRINTF1(str, a) 316 #define MPRINTF2(str, a, b) 317 #define MPRINTF3(str, a, b, c) 318 #endif /* DEBUGGING_MEM */ 319 320 /* Simple message to indicate that the bootops pointer has been zeroed */ 321 #ifdef DEBUG 322 static int bootops_gone_on = 0; 323 #define BOOTOPS_GONE() \ 324 if (bootops_gone_on) \ 325 prom_printf("The bootops vec is zeroed now!\n"); 326 #else 327 #define BOOTOPS_GONE() 328 #endif /* DEBUG */ 329 330 /* 331 * Monitor pages may not be where this says they are. 332 * and the debugger may not be there either. 333 * 334 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 335 * 336 * Physical memory layout 337 * (not necessarily contiguous) 338 * (THIS IS SOMEWHAT WRONG) 339 * /-----------------------\ 340 * | monitor pages | 341 * availmem -|-----------------------| 342 * | | 343 * | page pool | 344 * | | 345 * |-----------------------| 346 * | configured tables | 347 * | buffers | 348 * firstaddr -|-----------------------| 349 * | hat data structures | 350 * |-----------------------| 351 * | kernel data, bss | 352 * |-----------------------| 353 * | interrupt stack | 354 * |-----------------------| 355 * | kernel text (RO) | 356 * |-----------------------| 357 * | trap table (4k) | 358 * |-----------------------| 359 * page 1 | panicbuf | 360 * |-----------------------| 361 * page 0 | reclaimed | 362 * |_______________________| 363 * 364 * 365 * 366 * Kernel's Virtual Memory Layout. 367 * /-----------------------\ 368 * 0xFFFFFFFF.FFFFFFFF -| |- 369 * | OBP's virtual page | 370 * | tables | 371 * 0xFFFFFFFC.00000000 -|-----------------------|- 372 * : : 373 * : : 374 * 0xFFFFFE00.00000000 -|-----------------------|- 375 * | | Ultrasparc I/II support 376 * | segkpm segment | up to 2TB of physical 377 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 378 * | | 379 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 380 * : : 381 * : : 382 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 383 * | | ^ 384 * | UltraSPARC I/II call | | 385 * | bug requires an extra | | 386 * | 4 GB of space between | | 387 * | hole and used RAM | | 388 * | | | 389 * 0xFFFFF800.00000000 -|-----------------------|- | 390 * | | | 391 * | Virtual Address Hole | UltraSPARC 392 * | on UltraSPARC I/II | I/II * ONLY * 393 * | | | 394 * 0x00000800.00000000 -|-----------------------|- | 395 * | | | 396 * | UltraSPARC I/II call | | 397 * | bug requires an extra | | 398 * | 4 GB of space between | | 399 * | hole and used RAM | | 400 * | | v 401 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 402 * : : ^ 403 * : : | 404 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 405 * | | | 406 * | 64-bit kernel ONLY | | 407 * | | | 408 * | kmem64 segment | | 409 * | | | 410 * | (Relocated extra HME | Approximately 411 * | block allocations, | 1 TB of virtual 412 * | memnode freelists, | address space 413 * | HME hash buckets, | | 414 * | mml_table, kpmp_table,| | 415 * | page_t array and | | 416 * | hashblock pool to | | 417 * | avoid hard-coded | | 418 * | 32-bit vaddr | | 419 * | limitations) | | 420 * | | v 421 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 422 * | | 423 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 424 * | | 425 * 0x00000300.00000000 -|-----------------------|- SYSBASE 426 * : : 427 * : : 428 * -|-----------------------|- 429 * | | 430 * | segmap segment | SEGMAPSIZE (1/8th physmem, 431 * | | 256G MAX) 432 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 433 * : : 434 * : : 435 * -|-----------------------|- 436 * | | 437 * | segkp | SEGKPSIZE (2GB) 438 * | | 439 * | | 440 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 441 * | | 442 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 443 * | | (SEGKPBASE - 0x400000) 444 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 445 * | | (MEMSCRUBBASE - NCARGS) 446 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 447 * | | (ARGSBASE - PPMAPSIZE) 448 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 449 * | | 450 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 451 * | | 452 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 453 * : : 454 * : : 455 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 456 * | | 457 * | OBP | 458 * | | 459 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 460 * | kmdb | 461 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 462 * : : 463 * : : 464 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 465 * | | 466 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 467 * | | ~64MB) 468 * 0x00000000.78002000 -|-----------------------| 469 * | panicbuf | 470 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 471 * : : 472 * : : 473 * | | 474 * |-----------------------|- econtig32 475 * | vm structures | 476 * 0x00000000.01C00000 |-----------------------|- nalloc_end 477 * | TSBs | 478 * |-----------------------|- end/nalloc_base 479 * | kernel data & bss | 480 * 0x00000000.01800000 -|-----------------------| 481 * : nucleus text hole : 482 * 0x00000000.01400000 -|-----------------------| 483 * : : 484 * |-----------------------| 485 * | module text | 486 * |-----------------------|- e_text/modtext 487 * | kernel text | 488 * |-----------------------| 489 * | trap table (48k) | 490 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 491 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 492 * |-----------------------| 493 * | | 494 * | invalid | 495 * | | 496 * 0x00000000.00000000 _|_______________________| 497 * 498 * 499 * 500 * 32-bit User Virtual Memory Layout. 501 * /-----------------------\ 502 * | | 503 * | invalid | 504 * | | 505 * 0xFFC00000 -|-----------------------|- USERLIMIT 506 * | user stack | 507 * : : 508 * : : 509 * : : 510 * | user data | 511 * -|-----------------------|- 512 * | user text | 513 * 0x00002000 -|-----------------------|- 514 * | invalid | 515 * 0x00000000 _|_______________________| 516 * 517 * 518 * 519 * 64-bit User Virtual Memory Layout. 520 * /-----------------------\ 521 * | | 522 * | invalid | 523 * | | 524 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 525 * | user stack | 526 * : : 527 * : : 528 * : : 529 * | user data | 530 * -|-----------------------|- 531 * | user text | 532 * 0x00000000.00100000 -|-----------------------|- 533 * | invalid | 534 * 0x00000000.00000000 _|_______________________| 535 */ 536 537 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 538 extern uint64_t ecache_flush_address(void); 539 540 #pragma weak load_platform_modules 541 #pragma weak starcat_startup_memlist 542 #pragma weak ecache_init_scrub_flush_area 543 #pragma weak ecache_flush_address 544 545 546 /* 547 * By default the DR Cage is enabled for maximum OS 548 * MPSS performance. Users needing to disable the cage mechanism 549 * can set this variable to zero via /etc/system. 550 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 551 * will result in loss of DR functionality. 552 * Platforms wishing to disable kernel Cage by default 553 * should do so in their set_platform_defaults() routine. 554 */ 555 int kernel_cage_enable = 1; 556 557 static void 558 setup_cage_params(void) 559 { 560 void (*func)(void); 561 562 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 563 if (func != NULL) { 564 (*func)(); 565 return; 566 } 567 568 if (kernel_cage_enable == 0) { 569 return; 570 } 571 kcage_range_lock(); 572 if (kcage_range_init(phys_avail, 1) == 0) { 573 kcage_init(total_pages / 256); 574 } 575 kcage_range_unlock(); 576 577 if (kcage_on) { 578 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 579 } else { 580 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 581 } 582 583 } 584 585 /* 586 * Machine-dependent startup code 587 */ 588 void 589 startup(void) 590 { 591 startup_init(); 592 if (&startup_platform) 593 startup_platform(); 594 startup_memlist(); 595 startup_modules(); 596 setup_cage_params(); 597 startup_bop_gone(); 598 startup_vm(); 599 startup_end(); 600 } 601 602 struct regs sync_reg_buf; 603 uint64_t sync_tt; 604 605 void 606 sync_handler(void) 607 { 608 struct trap_info ti; 609 int i; 610 611 /* 612 * Prevent trying to talk to the other CPUs since they are 613 * sitting in the prom and won't reply. 614 */ 615 for (i = 0; i < NCPU; i++) { 616 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 617 cpu[i]->cpu_flags &= ~CPU_READY; 618 cpu[i]->cpu_flags |= CPU_QUIESCED; 619 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 620 } 621 } 622 623 /* 624 * We've managed to get here without going through the 625 * normal panic code path. Try and save some useful 626 * information. 627 */ 628 if (!panicstr && (curthread->t_panic_trap == NULL)) { 629 ti.trap_type = sync_tt; 630 ti.trap_regs = &sync_reg_buf; 631 ti.trap_addr = NULL; 632 ti.trap_mmu_fsr = 0x0; 633 634 curthread->t_panic_trap = &ti; 635 } 636 637 /* 638 * If we're re-entering the panic path, update the signature 639 * block so that the SC knows we're in the second part of panic. 640 */ 641 if (panicstr) 642 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 643 644 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 645 panic("sync initiated"); 646 } 647 648 649 static void 650 startup_init(void) 651 { 652 /* 653 * We want to save the registers while we're still in OBP 654 * so that we know they haven't been fiddled with since. 655 * (In principle, OBP can't change them just because it 656 * makes a callback, but we'd rather not depend on that 657 * behavior.) 658 */ 659 char sync_str[] = 660 "warning @ warning off : sync " 661 "%%tl-c %%tstate h# %p x! " 662 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 663 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 664 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 665 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 666 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 667 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 668 "%%y h# %p l! %%tl-c %%tt h# %p x! " 669 "sync ; warning !"; 670 671 /* 672 * 20 == num of %p substrings 673 * 16 == max num of chars %p will expand to. 674 */ 675 char bp[sizeof (sync_str) + 16 * 20]; 676 677 (void) check_boot_version(BOP_GETVERSION(bootops)); 678 679 /* 680 * Initialize ptl1 stack for the 1st CPU. 681 */ 682 ptl1_init_cpu(&cpu0); 683 684 /* 685 * Initialize the address map for cache consistent mappings 686 * to random pages; must be done after vac_size is set. 687 */ 688 ppmapinit(); 689 690 /* 691 * Initialize the PROM callback handler. 692 */ 693 init_vx_handler(); 694 695 /* 696 * have prom call sync_callback() to handle the sync and 697 * save some useful information which will be stored in the 698 * core file later. 699 */ 700 (void) sprintf((char *)bp, sync_str, 701 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 702 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 703 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 704 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 705 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 706 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 707 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 708 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 709 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 710 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 711 prom_interpret(bp, 0, 0, 0, 0, 0); 712 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 713 } 714 715 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 716 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 717 718 #define IVSIZE ((MAXIVNUM + 1) * sizeof (struct intr_vector)) 719 720 /* 721 * As OBP takes up some RAM when the system boots, pages will already be "lost" 722 * to the system and reflected in npages by the time we see it. 723 * 724 * We only want to allocate kernel structures in the 64-bit virtual address 725 * space on systems with enough RAM to make the overhead of keeping track of 726 * an extra kernel memory segment worthwhile. 727 * 728 * Since OBP has already performed its memory allocations by this point, if we 729 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 730 * memory in the 64-bit virtual address space; otherwise keep allocations 731 * contiguous with we've mapped so far in the 32-bit virtual address space. 732 */ 733 #define MINMOVE_RAM_MB ((size_t)1900) 734 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 735 736 pgcnt_t tune_npages = (pgcnt_t) 737 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 738 739 static void 740 startup_memlist(void) 741 { 742 size_t alloc_sz; 743 size_t ctrs_sz; 744 caddr_t alloc_base; 745 caddr_t ctrs_base, ctrs_end; 746 caddr_t memspace; 747 caddr_t va; 748 int memblocks = 0; 749 struct memlist *cur; 750 size_t syslimit = (size_t)SYSLIMIT; 751 size_t sysbase = (size_t)SYSBASE; 752 int alloc_alignsize = MMU_PAGESIZE; 753 extern void page_coloring_init(void); 754 755 /* 756 * Initialize enough of the system to allow kmem_alloc to work by 757 * calling boot to allocate its memory until the time that 758 * kvm_init is completed. The page structs are allocated after 759 * rounding up end to the nearest page boundary; the memsegs are 760 * initialized and the space they use comes from the kernel heap. 761 * With appropriate initialization, they can be reallocated later 762 * to a size appropriate for the machine's configuration. 763 * 764 * At this point, memory is allocated for things that will never 765 * need to be freed, this used to be "valloced". This allows a 766 * savings as the pages don't need page structures to describe 767 * them because them will not be managed by the vm system. 768 */ 769 770 /* 771 * We're loaded by boot with the following configuration (as 772 * specified in the sun4u/conf/Mapfile): 773 * 774 * text: 4 MB chunk aligned on a 4MB boundary 775 * data & bss: 4 MB chunk aligned on a 4MB boundary 776 * 777 * These two chunks will eventually be mapped by 2 locked 4MB 778 * ttes and will represent the nucleus of the kernel. This gives 779 * us some free space that is already allocated, some or all of 780 * which is made available to kernel module text. 781 * 782 * The free space in the data-bss chunk is used for nucleus 783 * allocatable data structures and we reserve it using the 784 * nalloc_base and nalloc_end variables. This space is currently 785 * being used for hat data structures required for tlb miss 786 * handling operations. We align nalloc_base to a l2 cache 787 * linesize because this is the line size the hardware uses to 788 * maintain cache coherency. 789 * 256K is carved out for module data. 790 */ 791 792 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 793 moddata = nalloc_base; 794 e_moddata = nalloc_base + MODDATA; 795 nalloc_base = e_moddata; 796 797 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 798 valloc_base = nalloc_base; 799 800 /* 801 * Calculate the start of the data segment. 802 */ 803 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 804 805 PRM_DEBUG(moddata); 806 PRM_DEBUG(nalloc_base); 807 PRM_DEBUG(nalloc_end); 808 PRM_DEBUG(sdata); 809 810 /* 811 * Remember any slop after e_text so we can give it to the modules. 812 */ 813 PRM_DEBUG(e_text); 814 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 815 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 816 panic("nucleus text overflow"); 817 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 818 modtext; 819 PRM_DEBUG(modtext); 820 PRM_DEBUG(modtext_sz); 821 822 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 823 &boot_physavail, &boot_physavail_len, 824 &boot_virtavail, &boot_virtavail_len); 825 /* 826 * Remember what the physically available highest page is 827 * so that dumpsys works properly, and find out how much 828 * memory is installed. 829 */ 830 installed_top_size_memlist_array(boot_physinstalled, 831 boot_physinstalled_len, &physmax, &physinstalled); 832 PRM_DEBUG(physinstalled); 833 PRM_DEBUG(physmax); 834 835 /* Fill out memory nodes config structure */ 836 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 837 838 /* 839 * Get the list of physically available memory to size 840 * the number of page structures needed. 841 */ 842 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 843 /* 844 * This first snap shot of npages can represent the pages used 845 * by OBP's text and data approximately. This is used in the 846 * the calculation of the kernel size 847 */ 848 obp_pages = physinstalled - npages; 849 850 851 /* 852 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 853 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 854 * 2MB) and any excess pages are put on physavail. The assumption is 855 * that small-memory systems will need more pages more than they'll 856 * need efficiently-mapped module texts. 857 */ 858 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 859 modtext_sz > MODTEXT_SM_CAP) { 860 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 861 modtext_sz = MODTEXT_SM_CAP; 862 } else 863 extra_etpg = 0; 864 PRM_DEBUG(extra_etpg); 865 PRM_DEBUG(modtext_sz); 866 extra_etva = modtext + modtext_sz; 867 PRM_DEBUG(extra_etva); 868 869 /* 870 * Account for any pages after e_text and e_data. 871 */ 872 npages += extra_etpg; 873 npages += mmu_btopr(nalloc_end - nalloc_base); 874 PRM_DEBUG(npages); 875 876 /* 877 * npages is the maximum of available physical memory possible. 878 * (ie. it will never be more than this) 879 */ 880 881 /* 882 * initialize the nucleus memory allocator. 883 */ 884 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 885 886 /* 887 * Allocate mmu fault status area from the nucleus data area. 888 */ 889 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 890 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 891 892 /* 893 * Allocate kernel TSBs from the nucleus data area. 894 */ 895 if (ndata_alloc_tsbs(&ndata, npages) != 0) 896 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 897 898 /* 899 * Allocate cpus structs from the nucleus data area. 900 */ 901 if (ndata_alloc_cpus(&ndata) != 0) 902 cmn_err(CE_PANIC, "no more nucleus memory after cpu alloc"); 903 904 /* 905 * Allocate dmv dispatch table from the nucleus data area. 906 */ 907 if (ndata_alloc_dmv(&ndata) != 0) 908 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 909 910 911 page_coloring_init(); 912 913 /* 914 * Allocate page_freelists bin headers for memnode 0 from the 915 * nucleus data area. 916 */ 917 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 918 cmn_err(CE_PANIC, 919 "no more nucleus memory after page free lists alloc"); 920 921 if (kpm_enable) { 922 kpm_init(); 923 /* 924 * kpm page space -- Update kpm_npages and make the 925 * same assumption about fragmenting as it is done 926 * for memseg_sz. 927 */ 928 kpm_npages_setup(memblocks + 4); 929 } 930 931 /* 932 * Allocate hat related structs from the nucleus data area. 933 */ 934 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 935 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 936 937 /* 938 * We want to do the BOP_ALLOCs before the real allocation of page 939 * structs in order to not have to allocate page structs for this 940 * memory. We need to calculate a virtual address because we want 941 * the page structs to come before other allocations in virtual address 942 * space. This is so some (if not all) of page structs can actually 943 * live in the nucleus. 944 */ 945 946 /* 947 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 948 * 949 * There are comments all over the SFMMU code warning of dire 950 * consequences if the TSBs are moved out of 32-bit space. This 951 * is largely because the asm code uses "sethi %hi(addr)"-type 952 * instructions which will not provide the expected result if the 953 * address is a 64-bit one. 954 * 955 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 956 */ 957 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 958 alloc_base = sfmmu_ktsb_alloc(alloc_base); 959 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 960 PRM_DEBUG(alloc_base); 961 962 /* 963 * Allocate IOMMU TSB array. We do this here so that the physical 964 * memory gets deducted from the PROM's physical memory list. 965 */ 966 alloc_base = iommu_tsb_init(alloc_base); 967 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 968 ecache_alignsize); 969 PRM_DEBUG(alloc_base); 970 971 /* 972 * Starcat needs its special structures assigned in 32-bit virtual 973 * address space because its probing routines execute FCode, and FCode 974 * can't handle 64-bit virtual addresses... 975 */ 976 if (&starcat_startup_memlist) { 977 alloc_base = starcat_startup_memlist(alloc_base); 978 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 979 ecache_alignsize); 980 PRM_DEBUG(alloc_base); 981 } 982 983 /* 984 * If we have enough memory, use 4M pages for alignment because it 985 * greatly reduces the number of TLB misses we take albeit at the cost 986 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 987 * allocation.) Still, the speedup on large memory systems (e.g. > 64 988 * GB) is quite noticeable, so it is worth the effort to do if we can. 989 * 990 * Note, however, that this speedup will only occur if the boot PROM 991 * uses the largest possible MMU page size possible to map memory 992 * requests that are properly aligned and sized (for example, a request 993 * for a multiple of 4MB of memory aligned to a 4MB boundary will 994 * result in a mapping using a 4MB MMU page.) 995 * 996 * Even then, the large page mappings will only speed things up until 997 * the startup process proceeds a bit further, as when 998 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 999 * kernel it remaps everything but the TSBs using 8K pages anyway... 1000 * 1001 * At some point in the future, sfmmu_map_prom_mappings() will be 1002 * rewritten to copy memory mappings to the kernel using the same MMU 1003 * page sizes the PROM used. When that occurs, if the PROM did use 1004 * large MMU pages to map memory, the alignment/sizing work we're 1005 * doing now should give us a nice extra performance boost, albeit at 1006 * the cost of greater RAM usage... 1007 */ 1008 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1009 MMU_PAGESIZE); 1010 1011 PRM_DEBUG(tune_npages); 1012 PRM_DEBUG(alloc_alignsize); 1013 1014 /* 1015 * Save off where the contiguous allocations to date have ended 1016 * in econtig32. 1017 */ 1018 econtig32 = alloc_base; 1019 PRM_DEBUG(econtig32); 1020 1021 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1022 cmn_err(CE_PANIC, "econtig32 too big"); 1023 1024 /* 1025 * To avoid memory allocation collisions in the 32-bit virtual address 1026 * space, make allocations from this point forward in 64-bit virtual 1027 * address space starting at syslimit and working up. Also use the 1028 * alignment specified by alloc_alignsize, as we may be able to save 1029 * ourselves TLB misses by using larger page sizes if they're 1030 * available. 1031 * 1032 * All this is needed because on large memory systems, the default 1033 * Solaris allocations will collide with SYSBASE32, which is hard 1034 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1035 * kernels, move the allocations to a location in the 64-bit virtual 1036 * address space space, allowing those structures to grow without 1037 * worry. 1038 * 1039 * On current CPUs we'll run out of physical memory address bits before 1040 * we need to worry about the allocations running into anything else in 1041 * VM or the virtual address holes on US-I and II, as there's currently 1042 * about 1 TB of addressable space before the US-I/II VA hole. 1043 */ 1044 kmem64_base = (caddr_t)syslimit; 1045 PRM_DEBUG(kmem64_base); 1046 1047 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1048 1049 /* 1050 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1051 * them here. 1052 */ 1053 if (khme_hash == NULL || uhme_hash == NULL) { 1054 /* 1055 * alloc_hme_buckets() will align alloc_base properly before 1056 * assigning the hash buckets, so we don't need to do it 1057 * before the call... 1058 */ 1059 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1060 1061 PRM_DEBUG(alloc_base); 1062 PRM_DEBUG(khme_hash); 1063 PRM_DEBUG(uhme_hash); 1064 } 1065 1066 /* 1067 * Allocate the remaining page freelists. NUMA systems can 1068 * have lots of page freelists, one per node, which quickly 1069 * outgrow the amount of nucleus memory available. 1070 */ 1071 if (max_mem_nodes > 1) { 1072 int mnode; 1073 caddr_t alloc_start = alloc_base; 1074 1075 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1076 alloc_base = alloc_page_freelists(mnode, alloc_base, 1077 ecache_alignsize); 1078 } 1079 1080 if (alloc_base > alloc_start) { 1081 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1082 alloc_alignsize); 1083 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1084 alloc_base - alloc_start, 1085 alloc_alignsize) != alloc_start) 1086 cmn_err(CE_PANIC, 1087 "Unable to alloc page freelists\n"); 1088 } 1089 1090 PRM_DEBUG(alloc_base); 1091 } 1092 1093 if (!mml_table) { 1094 size_t mmltable_sz; 1095 1096 /* 1097 * We need to allocate the mml_table here because there 1098 * was not enough space within the nucleus. 1099 */ 1100 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1101 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1102 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1103 alloc_alignsize); 1104 1105 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1106 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1107 panic("mml_table alloc failure"); 1108 1109 alloc_base += alloc_sz; 1110 PRM_DEBUG(mml_table); 1111 PRM_DEBUG(alloc_base); 1112 } 1113 1114 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1115 size_t kpmptable_sz; 1116 caddr_t table; 1117 1118 /* 1119 * We need to allocate either kpmp_table or kpmp_stable here 1120 * because there was not enough space within the nucleus. 1121 */ 1122 kpmptable_sz = (kpm_smallpages == 0) ? 1123 sizeof (kpm_hlk_t) * kpmp_table_sz : 1124 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1125 1126 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1127 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1128 alloc_alignsize); 1129 1130 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1131 alloc_alignsize); 1132 1133 if (table != alloc_base) 1134 panic("kpmp_table or kpmp_stable alloc failure"); 1135 1136 if (kpm_smallpages == 0) { 1137 kpmp_table = (kpm_hlk_t *)table; 1138 PRM_DEBUG(kpmp_table); 1139 } else { 1140 kpmp_stable = (kpm_shlk_t *)table; 1141 PRM_DEBUG(kpmp_stable); 1142 } 1143 1144 alloc_base += alloc_sz; 1145 PRM_DEBUG(alloc_base); 1146 } 1147 1148 if (&ecache_init_scrub_flush_area) { 1149 /* 1150 * Pass alloc_base directly, as the routine itself is 1151 * responsible for any special alignment requirements... 1152 */ 1153 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1154 PRM_DEBUG(alloc_base); 1155 } 1156 1157 /* 1158 * Take the most current snapshot we can by calling mem-update. 1159 */ 1160 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1161 &boot_physavail, &boot_physavail_len, 1162 &boot_virtavail, &boot_virtavail_len); 1163 1164 /* 1165 * Reset npages and memblocks based on boot_physavail list. 1166 */ 1167 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1168 PRM_DEBUG(npages); 1169 1170 /* 1171 * Account for extra memory after e_text. 1172 */ 1173 npages += extra_etpg; 1174 1175 /* 1176 * Calculate the largest free memory chunk in the nucleus data area. 1177 * We need to figure out if page structs can fit in there or not. 1178 * We also make sure enough page structs get created for any physical 1179 * memory we might be returning to the system. 1180 */ 1181 ndata_remain_sz = ndata_maxsize(&ndata); 1182 PRM_DEBUG(ndata_remain_sz); 1183 1184 pp_sz = sizeof (struct page) * npages; 1185 1186 /* 1187 * Here's a nice bit of code based on somewhat recursive logic: 1188 * 1189 * If the page array would fit within the nucleus, we want to 1190 * add npages to cover any extra memory we may be returning back 1191 * to the system. 1192 * 1193 * HOWEVER, the page array is sized by calculating the size of 1194 * (struct page * npages), as are the pagehash table, ctrs and 1195 * memseg_list, so the very act of performing the calculation below may 1196 * in fact make the array large enough that it no longer fits in the 1197 * nucleus, meaning there would now be a much larger area of the 1198 * nucleus free that should really be added to npages, which would 1199 * make the page array that much larger, and so on. 1200 * 1201 * This also ignores the memory possibly used in the nucleus for the 1202 * the page hash, ctrs and memseg list and the fact that whether they 1203 * fit there or not varies with the npages calculation below, but we 1204 * don't even factor them into the equation at this point; perhaps we 1205 * should or perhaps we should just take the approach that the few 1206 * extra pages we could add via this calculation REALLY aren't worth 1207 * the hassle... 1208 */ 1209 if (ndata_remain_sz > pp_sz) { 1210 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1211 1212 npages += mmu_btop(spare); 1213 1214 pp_sz = npages * sizeof (struct page); 1215 1216 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1217 } 1218 1219 /* 1220 * If physmem is patched to be non-zero, use it instead of 1221 * the monitor value unless physmem is larger than the total 1222 * amount of memory on hand. 1223 */ 1224 if (physmem == 0 || physmem > npages) 1225 physmem = npages; 1226 1227 /* 1228 * If pp_base is NULL that means the routines above have determined 1229 * the page array will not fit in the nucleus; we'll have to 1230 * BOP_ALLOC() ourselves some space for them. 1231 */ 1232 if (pp_base == NULL) { 1233 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1234 alloc_alignsize); 1235 1236 alloc_sz = roundup(pp_sz, alloc_alignsize); 1237 1238 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1239 alloc_base, alloc_sz, alloc_alignsize)) != 1240 (struct page *)alloc_base) 1241 panic("page alloc failure"); 1242 1243 alloc_base += alloc_sz; 1244 } 1245 1246 /* 1247 * The page structure hash table size is a power of 2 1248 * such that the average hash chain length is PAGE_HASHAVELEN. 1249 */ 1250 page_hashsz = npages / PAGE_HASHAVELEN; 1251 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1252 pagehash_sz = sizeof (struct page *) * page_hashsz; 1253 1254 /* 1255 * We want to TRY to fit the page structure hash table, 1256 * the page size free list counters, the memseg list and 1257 * and the kpm page space in the nucleus if possible. 1258 * 1259 * alloc_sz counts how much memory needs to be allocated by 1260 * BOP_ALLOC(). 1261 */ 1262 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1263 1264 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1265 1266 /* 1267 * Size up per page size free list counters. 1268 */ 1269 ctrs_sz = page_ctrs_sz(); 1270 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1271 1272 if (ctrs_base == NULL) 1273 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1274 1275 /* 1276 * The memseg list is for the chunks of physical memory that 1277 * will be managed by the vm system. The number calculated is 1278 * a guess as boot may fragment it more when memory allocations 1279 * are made before kphysm_init(). Currently, there are two 1280 * allocations before then, so we assume each causes fragmen- 1281 * tation, and add a couple more for good measure. 1282 */ 1283 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1284 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1285 1286 if (memseg_base == NULL) 1287 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1288 1289 1290 if (kpm_enable) { 1291 /* 1292 * kpm page space -- Update kpm_npages and make the 1293 * same assumption about fragmenting as it is done 1294 * for memseg_sz above. 1295 */ 1296 kpm_npages_setup(memblocks + 4); 1297 kpm_pp_sz = (kpm_smallpages == 0) ? 1298 kpm_npages * sizeof (kpm_page_t): 1299 kpm_npages * sizeof (kpm_spage_t); 1300 1301 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1302 ecache_alignsize); 1303 1304 if (kpm_pp_base == NULL) 1305 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1306 kpm_pp_sz; 1307 } 1308 1309 if (alloc_sz > 0) { 1310 uintptr_t bop_base; 1311 1312 /* 1313 * We need extra memory allocated through BOP_ALLOC. 1314 */ 1315 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1316 alloc_alignsize); 1317 1318 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1319 1320 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1321 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1322 panic("system page struct alloc failure"); 1323 1324 alloc_base += alloc_sz; 1325 1326 if (page_hash == NULL) { 1327 page_hash = (struct page **)bop_base; 1328 bop_base = roundup(bop_base + pagehash_sz, 1329 ecache_alignsize); 1330 } 1331 1332 if (ctrs_base == NULL) { 1333 ctrs_base = (caddr_t)bop_base; 1334 bop_base = roundup(bop_base + ctrs_sz, 1335 ecache_alignsize); 1336 } 1337 1338 if (memseg_base == NULL) { 1339 memseg_base = (struct memseg *)bop_base; 1340 bop_base = roundup(bop_base + memseg_sz, 1341 ecache_alignsize); 1342 } 1343 1344 if (kpm_enable && kpm_pp_base == NULL) { 1345 kpm_pp_base = (uintptr_t)bop_base; 1346 bop_base = roundup(bop_base + kpm_pp_sz, 1347 ecache_alignsize); 1348 } 1349 1350 ASSERT(bop_base <= (uintptr_t)alloc_base); 1351 } 1352 1353 /* 1354 * Initialize per page size free list counters. 1355 */ 1356 ctrs_end = page_ctrs_alloc(ctrs_base); 1357 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1358 1359 PRM_DEBUG(page_hash); 1360 PRM_DEBUG(memseg_base); 1361 PRM_DEBUG(kpm_pp_base); 1362 PRM_DEBUG(kpm_pp_sz); 1363 PRM_DEBUG(pp_base); 1364 PRM_DEBUG(pp_sz); 1365 PRM_DEBUG(alloc_base); 1366 1367 #ifdef TRAPTRACE 1368 /* 1369 * Allocate trap trace buffer last so as not to affect 1370 * the 4M alignments of the allocations above on V9 SPARCs... 1371 */ 1372 alloc_base = trap_trace_alloc(alloc_base); 1373 PRM_DEBUG(alloc_base); 1374 #endif /* TRAPTRACE */ 1375 1376 if (kmem64_base) { 1377 /* 1378 * Set the end of the kmem64 segment for V9 SPARCs, if 1379 * appropriate... 1380 */ 1381 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1382 alloc_alignsize); 1383 1384 PRM_DEBUG(kmem64_base); 1385 PRM_DEBUG(kmem64_end); 1386 } 1387 1388 /* 1389 * Allocate space for the interrupt vector table. 1390 */ 1391 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vector, 1392 IVSIZE, MMU_PAGESIZE); 1393 if (memspace != (caddr_t)intr_vector) 1394 panic("interrupt table allocation failure"); 1395 1396 /* 1397 * The memory lists from boot are allocated from the heap arena 1398 * so that later they can be freed and/or reallocated. 1399 */ 1400 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1401 panic("could not retrieve property \"extent\""); 1402 1403 /* 1404 * Between now and when we finish copying in the memory lists, 1405 * allocations happen so the space gets fragmented and the 1406 * lists longer. Leave enough space for lists twice as long 1407 * as what boot says it has now; roundup to a pagesize. 1408 * Also add space for the final phys-avail copy in the fixup 1409 * routine. 1410 */ 1411 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1412 roundup(IVSIZE, MMU_PAGESIZE)); 1413 memlist_sz *= 4; 1414 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1415 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1416 if (memspace == NULL) 1417 halt("Boot allocation failed."); 1418 1419 memlist = (struct memlist *)memspace; 1420 memlist_end = (char *)memspace + memlist_sz; 1421 1422 PRM_DEBUG(memlist); 1423 PRM_DEBUG(memlist_end); 1424 PRM_DEBUG(sysbase); 1425 PRM_DEBUG(syslimit); 1426 1427 kernelheap_init((void *)sysbase, (void *)syslimit, 1428 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1429 1430 /* 1431 * Take the most current snapshot we can by calling mem-update. 1432 */ 1433 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1434 &boot_physavail, &boot_physavail_len, 1435 &boot_virtavail, &boot_virtavail_len); 1436 1437 /* 1438 * Remove the space used by BOP_ALLOC from the kernel heap 1439 * plus the area actually used by the OBP (if any) 1440 * ignoring virtual addresses in virt_avail, above syslimit. 1441 */ 1442 virt_avail = memlist; 1443 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1444 1445 for (cur = virt_avail; cur->next; cur = cur->next) { 1446 uint64_t range_base, range_size; 1447 1448 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1449 continue; 1450 if (range_base >= (uint64_t)syslimit) 1451 break; 1452 /* 1453 * Limit the range to end at syslimit. 1454 */ 1455 range_size = MIN(cur->next->address, 1456 (uint64_t)syslimit) - range_base; 1457 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1458 0, 0, (void *)range_base, (void *)(range_base + range_size), 1459 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1460 } 1461 1462 phys_avail = memlist; 1463 (void) copy_physavail(boot_physavail, boot_physavail_len, 1464 &memlist, 0, 0); 1465 1466 /* 1467 * Add any extra memory after e_text to the phys_avail list, as long 1468 * as there's at least a page to add. 1469 */ 1470 if (extra_etpg) 1471 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1472 &memlist, &phys_avail); 1473 1474 /* 1475 * Add any extra memory after e_data to the phys_avail list as long 1476 * as there's at least a page to add. Usually, there isn't any, 1477 * since extra HME blocks typically get allocated there first before 1478 * using RAM elsewhere. 1479 */ 1480 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1481 nalloc_base = nalloc_end; 1482 ndata_remain_sz = nalloc_end - nalloc_base; 1483 1484 if (ndata_remain_sz >= MMU_PAGESIZE) 1485 memlist_add(va_to_pa(nalloc_base), 1486 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1487 1488 PRM_DEBUG(memlist); 1489 PRM_DEBUG(memlist_sz); 1490 PRM_DEBUG(memspace); 1491 1492 if ((caddr_t)memlist > (memspace + memlist_sz)) 1493 panic("memlist overflow"); 1494 1495 PRM_DEBUG(pp_base); 1496 PRM_DEBUG(memseg_base); 1497 PRM_DEBUG(npages); 1498 1499 /* 1500 * Initialize the page structures from the memory lists. 1501 */ 1502 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1503 1504 availrmem_initial = availrmem = freemem; 1505 PRM_DEBUG(availrmem); 1506 1507 /* 1508 * Some of the locks depend on page_hashsz being set! 1509 * kmem_init() depends on this; so, keep it here. 1510 */ 1511 page_lock_init(); 1512 1513 /* 1514 * Initialize kernel memory allocator. 1515 */ 1516 kmem_init(); 1517 1518 /* 1519 * Initialize bp_mapin(). 1520 */ 1521 bp_init(shm_alignment, HAT_STRICTORDER); 1522 1523 /* 1524 * Reserve space for panicbuf and intr_vector from the 32-bit heap 1525 */ 1526 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1527 panicbuf, panicbuf + PANICBUFSIZE, 1528 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1529 1530 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1531 intr_vector, (caddr_t)intr_vector + IVSIZE, 1532 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1533 1534 mem_config_init(); 1535 } 1536 1537 static void 1538 startup_modules(void) 1539 { 1540 int proplen, nhblk1, nhblk8; 1541 size_t nhblksz; 1542 pgcnt_t hblk_pages, pages_per_hblk; 1543 size_t hme8blk_sz, hme1blk_sz; 1544 1545 /* 1546 * Log any optional messages from the boot program 1547 */ 1548 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1549 if (proplen > 0) { 1550 char *msg; 1551 size_t len = (size_t)proplen; 1552 1553 msg = kmem_zalloc(len, KM_SLEEP); 1554 (void) BOP_GETPROP(bootops, "boot-message", msg); 1555 cmn_err(CE_CONT, "?%s\n", msg); 1556 kmem_free(msg, len); 1557 } 1558 1559 /* 1560 * Let the platforms have a chance to change default 1561 * values before reading system file. 1562 */ 1563 if (&set_platform_defaults) 1564 set_platform_defaults(); 1565 1566 /* 1567 * Calculate default settings of system parameters based upon 1568 * maxusers, yet allow to be overridden via the /etc/system file. 1569 */ 1570 param_calc(0); 1571 1572 mod_setup(); 1573 1574 /* 1575 * If this is a positron, complain and halt. 1576 */ 1577 if (&iam_positron && iam_positron()) { 1578 cmn_err(CE_WARN, "This hardware platform is not supported" 1579 " by this release of Solaris.\n"); 1580 #ifdef DEBUG 1581 prom_enter_mon(); /* Type 'go' to resume */ 1582 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1583 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1584 1585 #else /* DEBUG */ 1586 halt(0); 1587 #endif /* DEBUG */ 1588 } 1589 1590 /* 1591 * If we are running firmware that isn't 64-bit ready 1592 * then complain and halt. 1593 */ 1594 do_prom_version_check(); 1595 1596 /* 1597 * Initialize system parameters 1598 */ 1599 param_init(); 1600 1601 /* 1602 * maxmem is the amount of physical memory we're playing with. 1603 */ 1604 maxmem = physmem; 1605 1606 /* Set segkp limits. */ 1607 ncbase = (caddr_t)SEGDEBUGBASE; 1608 ncend = (caddr_t)SEGDEBUGBASE; 1609 1610 /* 1611 * Initialize the hat layer. 1612 */ 1613 hat_init(); 1614 1615 /* 1616 * Initialize segment management stuff. 1617 */ 1618 seg_init(); 1619 1620 /* 1621 * Create the va>tte handler, so the prom can understand 1622 * kernel translations. The handler is installed later, just 1623 * as we are about to take over the trap table from the prom. 1624 */ 1625 create_va_to_tte(); 1626 1627 /* 1628 * Load the forthdebugger (optional) 1629 */ 1630 forthdebug_init(); 1631 1632 /* 1633 * Create OBP node for console input callbacks 1634 * if it is needed. 1635 */ 1636 startup_create_input_node(); 1637 1638 if (modloadonly("fs", "specfs") == -1) 1639 halt("Can't load specfs"); 1640 1641 if (modloadonly("fs", "devfs") == -1) 1642 halt("Can't load devfs"); 1643 1644 if (modloadonly("misc", "swapgeneric") == -1) 1645 halt("Can't load swapgeneric"); 1646 1647 dispinit(); 1648 1649 /* 1650 * Infer meanings to the members of the idprom buffer. 1651 */ 1652 parse_idprom(); 1653 1654 /* Read cluster configuration data. */ 1655 clconf_init(); 1656 1657 setup_ddi(); 1658 1659 /* 1660 * Lets take this opportunity to load the root device. 1661 */ 1662 if (loadrootmodules() != 0) 1663 debug_enter("Can't load the root filesystem"); 1664 1665 /* 1666 * Load tod driver module for the tod part found on this system. 1667 * Recompute the cpu frequency/delays based on tod as tod part 1668 * tends to keep time more accurately. 1669 */ 1670 if (&load_tod_module) 1671 load_tod_module(); 1672 1673 /* 1674 * Allow platforms to load modules which might 1675 * be needed after bootops are gone. 1676 */ 1677 if (&load_platform_modules) 1678 load_platform_modules(); 1679 1680 setcpudelay(); 1681 1682 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1683 &boot_physavail, &boot_physavail_len, 1684 &boot_virtavail, &boot_virtavail_len); 1685 1686 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1687 1688 /* 1689 * Calculation and allocation of hmeblks needed to remap 1690 * the memory allocated by PROM till now: 1691 * 1692 * (1) calculate how much virtual memory has been bop_alloc'ed. 1693 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1694 * (3) calculate number of hme8blk's needed to remap this memory 1695 * (4) calculate amount of memory that's consumed by these hme8blk's 1696 * (5) add memory calculated in steps (2) and (4) above. 1697 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1698 * (7) calculate number of hme8blk's needed to remap this memory 1699 * (8) calculate amount of memory that's consumed by these hme8blk's 1700 * (9) allocate additional hme1blk's to hold large mappings. 1701 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1702 * to remap everything with 4M mappings. 1703 * (10) account for partially used hblk8's due to non-64K aligned 1704 * PROM mapping entries. 1705 * (11) add memory calculated in steps (8), (9), and (10) above. 1706 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1707 * is not ready yet, this gets bop_alloc'ed. 1708 * (13) there will be very few bop_alloc's after this point before 1709 * trap table takes over 1710 */ 1711 1712 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1713 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1714 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1715 1716 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1717 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1718 nhblk8 = bop_alloc_pages / pages_per_hblk; 1719 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1720 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1721 bop_alloc_pages += hblk_pages; 1722 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1723 nhblk8 = bop_alloc_pages / pages_per_hblk; 1724 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1725 if (nhblk1 < hblk1_min) 1726 nhblk1 = hblk1_min; 1727 if (nhblk8 < hblk8_min) 1728 nhblk8 = hblk8_min; 1729 1730 /* 1731 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1732 * boundary, the number of hblk8's needed to map the entries in the 1733 * boot_virtavail list needs to be adjusted to take this into 1734 * consideration. Thus, we need to add additional hblk8's since it 1735 * is possible that an hblk8 will not have all 8 slots used due to 1736 * alignment constraints. Since there were boot_virtavail_len entries 1737 * in that list, we need to add that many hblk8's to the number 1738 * already calculated to make sure we don't underestimate. 1739 */ 1740 nhblk8 += boot_virtavail_len; 1741 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1742 1743 /* Allocate in pagesize chunks */ 1744 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1745 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1746 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1747 } 1748 1749 static void 1750 startup_bop_gone(void) 1751 { 1752 extern int bop_io_quiesced; 1753 1754 /* 1755 * Call back into boot and release boots resources. 1756 */ 1757 BOP_QUIESCE_IO(bootops); 1758 bop_io_quiesced = 1; 1759 1760 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1761 &boot_physavail, &boot_physavail_len, 1762 &boot_virtavail, &boot_virtavail_len); 1763 /* 1764 * Copy physinstalled list into kernel space. 1765 */ 1766 phys_install = memlist; 1767 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1768 1769 /* 1770 * setup physically contiguous area twice as large as the ecache. 1771 * this is used while doing displacement flush of ecaches 1772 */ 1773 if (&ecache_flush_address) { 1774 ecache_flushaddr = ecache_flush_address(); 1775 if (ecache_flushaddr == (uint64_t)-1) { 1776 cmn_err(CE_PANIC, 1777 "startup: no memory to set ecache_flushaddr"); 1778 } 1779 } 1780 1781 /* 1782 * Virtual available next. 1783 */ 1784 ASSERT(virt_avail != NULL); 1785 memlist_free_list(virt_avail); 1786 virt_avail = memlist; 1787 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1788 1789 /* 1790 * Last chance to ask our booter questions .. 1791 */ 1792 } 1793 1794 1795 /* 1796 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1797 * allocations have been performed. We can't call it in startup_bop_gone 1798 * since later operations can cause obp to allocate more memory. 1799 */ 1800 void 1801 startup_fixup_physavail(void) 1802 { 1803 struct memlist *cur; 1804 1805 /* 1806 * take the most current snapshot we can by calling mem-update 1807 */ 1808 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1809 &boot_physavail, &boot_physavail_len, 1810 &boot_virtavail, &boot_virtavail_len); 1811 1812 /* 1813 * Copy phys_avail list, again. 1814 * Both the kernel/boot and the prom have been allocating 1815 * from the original list we copied earlier. 1816 */ 1817 cur = memlist; 1818 (void) copy_physavail(boot_physavail, boot_physavail_len, 1819 &memlist, 0, 0); 1820 1821 /* 1822 * Add any extra memory after e_text we added to the phys_avail list 1823 * back to the old list. 1824 */ 1825 if (extra_etpg) 1826 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1827 &memlist, &cur); 1828 if (ndata_remain_sz >= MMU_PAGESIZE) 1829 memlist_add(va_to_pa(nalloc_base), 1830 (uint64_t)ndata_remain_sz, &memlist, &cur); 1831 1832 /* 1833 * There isn't any bounds checking on the memlist area 1834 * so ensure it hasn't overgrown. 1835 */ 1836 if ((caddr_t)memlist > (caddr_t)memlist_end) 1837 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1838 1839 /* 1840 * The kernel removes the pages that were allocated for it from 1841 * the freelist, but we now have to find any -extra- pages that 1842 * the prom has allocated for it's own book-keeping, and remove 1843 * them from the freelist too. sigh. 1844 */ 1845 fix_prom_pages(phys_avail, cur); 1846 1847 ASSERT(phys_avail != NULL); 1848 memlist_free_list(phys_avail); 1849 phys_avail = cur; 1850 1851 /* 1852 * We're done with boot. Just after this point in time, boot 1853 * gets unmapped, so we can no longer rely on its services. 1854 * Zero the bootops to indicate this fact. 1855 */ 1856 bootops = (struct bootops *)NULL; 1857 BOOTOPS_GONE(); 1858 } 1859 1860 static void 1861 startup_vm(void) 1862 { 1863 size_t i; 1864 struct segmap_crargs a; 1865 struct segkpm_crargs b; 1866 1867 uint64_t avmem; 1868 caddr_t va; 1869 pgcnt_t max_phys_segkp; 1870 int mnode; 1871 1872 extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg; 1873 1874 /* 1875 * get prom's mappings, create hments for them and switch 1876 * to the kernel context. 1877 */ 1878 hat_kern_setup(); 1879 1880 /* 1881 * Take over trap table 1882 */ 1883 setup_trap_table(); 1884 1885 /* 1886 * Install the va>tte handler, so that the prom can handle 1887 * misses and understand the kernel table layout in case 1888 * we need call into the prom. 1889 */ 1890 install_va_to_tte(); 1891 1892 /* 1893 * Set a flag to indicate that the tba has been taken over. 1894 */ 1895 tba_taken_over = 1; 1896 1897 /* initialize MMU primary context register */ 1898 mmu_init_kcontext(); 1899 1900 /* 1901 * The boot cpu can now take interrupts, x-calls, x-traps 1902 */ 1903 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1904 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1905 1906 /* 1907 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1908 */ 1909 tbr_wr_addr_inited = 1; 1910 1911 /* 1912 * Initialize VM system, and map kernel address space. 1913 */ 1914 kvm_init(); 1915 1916 /* 1917 * XXX4U: previously, we initialized and turned on 1918 * the caches at this point. But of course we have 1919 * nothing to do, as the prom has already done this 1920 * for us -- main memory must be E$able at all times. 1921 */ 1922 1923 /* 1924 * If the following is true, someone has patched 1925 * phsymem to be less than the number of pages that 1926 * the system actually has. Remove pages until system 1927 * memory is limited to the requested amount. Since we 1928 * have allocated page structures for all pages, we 1929 * correct the amount of memory we want to remove 1930 * by the size of the memory used to hold page structures 1931 * for the non-used pages. 1932 */ 1933 if (physmem < npages) { 1934 pgcnt_t diff, off; 1935 struct page *pp; 1936 struct seg kseg; 1937 1938 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1939 1940 off = 0; 1941 diff = npages - physmem; 1942 diff -= mmu_btopr(diff * sizeof (struct page)); 1943 kseg.s_as = &kas; 1944 while (diff--) { 1945 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1946 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1947 &kseg, (caddr_t)off); 1948 if (pp == NULL) 1949 cmn_err(CE_PANIC, "limited physmem too much!"); 1950 page_io_unlock(pp); 1951 page_downgrade(pp); 1952 availrmem--; 1953 off += MMU_PAGESIZE; 1954 } 1955 } 1956 1957 /* 1958 * When printing memory, show the total as physmem less 1959 * that stolen by a debugger. 1960 */ 1961 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1962 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1963 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1964 1965 avmem = (uint64_t)freemem << PAGESHIFT; 1966 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1967 1968 /* For small memory systems disable automatic large pages. */ 1969 if (physmem < auto_lpg_min_physmem) { 1970 exec_lpg_disable = 1; 1971 use_brk_lpg = 0; 1972 use_stk_lpg = 0; 1973 use_zmap_lpg = 0; 1974 } 1975 1976 /* 1977 * Perform platform specific freelist processing 1978 */ 1979 if (&plat_freelist_process) { 1980 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1981 if (mem_node_config[mnode].exists) 1982 plat_freelist_process(mnode); 1983 } 1984 1985 /* 1986 * Initialize the segkp segment type. We position it 1987 * after the configured tables and buffers (whose end 1988 * is given by econtig) and before V_WKBASE_ADDR. 1989 * Also in this area is segkmap (size SEGMAPSIZE). 1990 */ 1991 1992 /* XXX - cache alignment? */ 1993 va = (caddr_t)SEGKPBASE; 1994 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1995 1996 max_phys_segkp = (physmem * 2); 1997 1998 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1999 segkpsize = btop(SEGKPDEFSIZE); 2000 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2001 "segkpsize has been reset to %ld pages", segkpsize); 2002 } 2003 2004 i = ptob(MIN(segkpsize, max_phys_segkp)); 2005 2006 rw_enter(&kas.a_lock, RW_WRITER); 2007 if (seg_attach(&kas, va, i, segkp) < 0) 2008 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2009 if (segkp_create(segkp) != 0) 2010 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2011 rw_exit(&kas.a_lock); 2012 2013 /* 2014 * kpm segment 2015 */ 2016 segmap_kpm = kpm_enable && 2017 segmap_kpm && PAGESIZE == MAXBSIZE; 2018 2019 if (kpm_enable) { 2020 rw_enter(&kas.a_lock, RW_WRITER); 2021 2022 /* 2023 * The segkpm virtual range range is larger than the 2024 * actual physical memory size and also covers gaps in 2025 * the physical address range for the following reasons: 2026 * . keep conversion between segkpm and physical addresses 2027 * simple, cheap and unambiguous. 2028 * . avoid extension/shrink of the the segkpm in case of DR. 2029 * . avoid complexity for handling of virtual addressed 2030 * caches, segkpm and the regular mapping scheme must be 2031 * kept in sync wrt. the virtual color of mapped pages. 2032 * Any accesses to virtual segkpm ranges not backed by 2033 * physical memory will fall through the memseg pfn hash 2034 * and will be handled in segkpm_fault. 2035 * Additional kpm_size spaces needed for vac alias prevention. 2036 */ 2037 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2038 segkpm) < 0) 2039 cmn_err(CE_PANIC, "cannot attach segkpm"); 2040 2041 b.prot = PROT_READ | PROT_WRITE; 2042 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2043 2044 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2045 panic("segkpm_create segkpm"); 2046 2047 rw_exit(&kas.a_lock); 2048 } 2049 2050 /* 2051 * Now create generic mapping segment. This mapping 2052 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2053 * virtual address is greater than the amount of free 2054 * memory that is available, then we trim back the 2055 * segment size to that amount 2056 */ 2057 va = (caddr_t)SEGMAPBASE; 2058 2059 /* 2060 * 1201049: segkmap base address must be MAXBSIZE aligned 2061 */ 2062 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2063 2064 /* 2065 * Set size of segmap to percentage of freemem at boot, 2066 * but stay within the allowable range 2067 * Note we take percentage before converting from pages 2068 * to bytes to avoid an overflow on 32-bit kernels. 2069 */ 2070 i = mmu_ptob((freemem * segmap_percent) / 100); 2071 2072 if (i < MINMAPSIZE) 2073 i = MINMAPSIZE; 2074 2075 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2076 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2077 2078 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2079 2080 rw_enter(&kas.a_lock, RW_WRITER); 2081 if (seg_attach(&kas, va, i, segkmap) < 0) 2082 cmn_err(CE_PANIC, "cannot attach segkmap"); 2083 2084 a.prot = PROT_READ | PROT_WRITE; 2085 a.shmsize = shm_alignment; 2086 a.nfreelist = 0; /* use segmap driver defaults */ 2087 2088 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2089 panic("segmap_create segkmap"); 2090 rw_exit(&kas.a_lock); 2091 2092 segdev_init(); 2093 } 2094 2095 static void 2096 startup_end(void) 2097 { 2098 if ((caddr_t)memlist > (caddr_t)memlist_end) 2099 panic("memlist overflow 2"); 2100 memlist_free_block((caddr_t)memlist, 2101 ((caddr_t)memlist_end - (caddr_t)memlist)); 2102 memlist = NULL; 2103 2104 /* enable page_relocation since OBP is now done */ 2105 page_relocate_ready = 1; 2106 2107 /* 2108 * Perform tasks that get done after most of the VM 2109 * initialization has been done but before the clock 2110 * and other devices get started. 2111 */ 2112 kern_setup1(); 2113 2114 /* 2115 * Intialize the VM arenas for allocating physically 2116 * contiguus memory chunk for interrupt queues snd 2117 * allocate/register boot cpu's queues, if any and 2118 * allocate dump buffer for sun4v systems to store 2119 * extra crash information during crash dump 2120 */ 2121 contig_mem_init(); 2122 mach_descrip_init(); 2123 cpu_intrq_setup(CPU); 2124 cpu_intrq_register(CPU); 2125 mach_dump_buffer_init(); 2126 2127 /* 2128 * Initialize interrupt related stuff 2129 */ 2130 init_intr_threads(CPU); 2131 2132 (void) splzs(); /* allow hi clock ints but not zs */ 2133 2134 /* 2135 * Initialize errors. 2136 */ 2137 error_init(); 2138 2139 /* 2140 * Note that we may have already used kernel bcopy before this 2141 * point - but if you really care about this, adb the use_hw_* 2142 * variables to 0 before rebooting. 2143 */ 2144 mach_hw_copy_limit(); 2145 2146 /* 2147 * Install the "real" preemption guards before DDI services 2148 * are available. 2149 */ 2150 (void) prom_set_preprom(kern_preprom); 2151 (void) prom_set_postprom(kern_postprom); 2152 CPU->cpu_m.mutex_ready = 1; 2153 2154 /* 2155 * Initialize segnf (kernel support for non-faulting loads). 2156 */ 2157 segnf_init(); 2158 2159 /* 2160 * Configure the root devinfo node. 2161 */ 2162 configure(); /* set up devices */ 2163 mach_cpu_halt_idle(); 2164 } 2165 2166 2167 void 2168 post_startup(void) 2169 { 2170 #ifdef PTL1_PANIC_DEBUG 2171 extern void init_ptl1_thread(void); 2172 #endif /* PTL1_PANIC_DEBUG */ 2173 extern void abort_sequence_init(void); 2174 2175 /* 2176 * Set the system wide, processor-specific flags to be passed 2177 * to userland via the aux vector for performance hints and 2178 * instruction set extensions. 2179 */ 2180 bind_hwcap(); 2181 2182 /* 2183 * Startup memory scrubber (if any) 2184 */ 2185 mach_memscrub(); 2186 2187 /* 2188 * Allocate soft interrupt to handle abort sequence. 2189 */ 2190 abort_sequence_init(); 2191 2192 /* 2193 * Configure the rest of the system. 2194 * Perform forceloading tasks for /etc/system. 2195 */ 2196 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2197 /* 2198 * ON4.0: Force /proc module in until clock interrupt handle fixed 2199 * ON4.0: This must be fixed or restated in /etc/systems. 2200 */ 2201 (void) modload("fs", "procfs"); 2202 2203 if (&load_platform_drivers) 2204 load_platform_drivers(); 2205 2206 /* load vis simulation module, if we are running w/fpu off */ 2207 if (!fpu_exists) { 2208 if (modload("misc", "vis") == -1) 2209 halt("Can't load vis"); 2210 } 2211 2212 mach_fpras(); 2213 2214 maxmem = freemem; 2215 2216 #ifdef PTL1_PANIC_DEBUG 2217 init_ptl1_thread(); 2218 #endif /* PTL1_PANIC_DEBUG */ 2219 } 2220 2221 #ifdef PTL1_PANIC_DEBUG 2222 int ptl1_panic_test = 0; 2223 int ptl1_panic_xc_one_test = 0; 2224 int ptl1_panic_xc_all_test = 0; 2225 int ptl1_panic_xt_one_test = 0; 2226 int ptl1_panic_xt_all_test = 0; 2227 kthread_id_t ptl1_thread_p = NULL; 2228 kcondvar_t ptl1_cv; 2229 kmutex_t ptl1_mutex; 2230 int ptl1_recurse_count_threshold = 0x40; 2231 int ptl1_recurse_trap_threshold = 0x3d; 2232 extern void ptl1_recurse(int, int); 2233 extern void ptl1_panic_xt(int, int); 2234 2235 /* 2236 * Called once per second by timeout() to wake up 2237 * the ptl1_panic thread to see if it should cause 2238 * a trap to the ptl1_panic() code. 2239 */ 2240 /* ARGSUSED */ 2241 static void 2242 ptl1_wakeup(void *arg) 2243 { 2244 mutex_enter(&ptl1_mutex); 2245 cv_signal(&ptl1_cv); 2246 mutex_exit(&ptl1_mutex); 2247 } 2248 2249 /* 2250 * ptl1_panic cross call function: 2251 * Needed because xc_one() and xc_some() can pass 2252 * 64 bit args but ptl1_recurse() expects ints. 2253 */ 2254 static void 2255 ptl1_panic_xc(void) 2256 { 2257 ptl1_recurse(ptl1_recurse_count_threshold, 2258 ptl1_recurse_trap_threshold); 2259 } 2260 2261 /* 2262 * The ptl1 thread waits for a global flag to be set 2263 * and uses the recurse thresholds to set the stack depth 2264 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2265 * or indirectly via the cross call and cross trap functions. 2266 * 2267 * This is useful testing stack overflows and normal 2268 * ptl1_panic() states with a know stack frame. 2269 * 2270 * ptl1_recurse() is an asm function in ptl1_panic.s that 2271 * sets the {In, Local, Out, and Global} registers to a 2272 * know state on the stack and just prior to causing a 2273 * test ptl1_panic trap. 2274 */ 2275 static void 2276 ptl1_thread(void) 2277 { 2278 mutex_enter(&ptl1_mutex); 2279 while (ptl1_thread_p) { 2280 cpuset_t other_cpus; 2281 int cpu_id; 2282 int my_cpu_id; 2283 int target_cpu_id; 2284 int target_found; 2285 2286 if (ptl1_panic_test) { 2287 ptl1_recurse(ptl1_recurse_count_threshold, 2288 ptl1_recurse_trap_threshold); 2289 } 2290 2291 /* 2292 * Find potential targets for x-call and x-trap, 2293 * if any exist while preempt is disabled we 2294 * start a ptl1_panic if requested via a 2295 * globals. 2296 */ 2297 kpreempt_disable(); 2298 my_cpu_id = CPU->cpu_id; 2299 other_cpus = cpu_ready_set; 2300 CPUSET_DEL(other_cpus, CPU->cpu_id); 2301 target_found = 0; 2302 if (!CPUSET_ISNULL(other_cpus)) { 2303 /* 2304 * Pick the first one 2305 */ 2306 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2307 if (cpu_id == my_cpu_id) 2308 continue; 2309 2310 if (CPU_XCALL_READY(cpu_id)) { 2311 target_cpu_id = cpu_id; 2312 target_found = 1; 2313 break; 2314 } 2315 } 2316 ASSERT(target_found); 2317 2318 if (ptl1_panic_xc_one_test) { 2319 xc_one(target_cpu_id, 2320 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2321 } 2322 if (ptl1_panic_xc_all_test) { 2323 xc_some(other_cpus, 2324 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2325 } 2326 if (ptl1_panic_xt_one_test) { 2327 xt_one(target_cpu_id, 2328 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2329 } 2330 if (ptl1_panic_xt_all_test) { 2331 xt_some(other_cpus, 2332 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2333 } 2334 } 2335 kpreempt_enable(); 2336 (void) timeout(ptl1_wakeup, NULL, hz); 2337 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2338 } 2339 mutex_exit(&ptl1_mutex); 2340 } 2341 2342 /* 2343 * Called during early startup to create the ptl1_thread 2344 */ 2345 void 2346 init_ptl1_thread(void) 2347 { 2348 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2349 &p0, TS_RUN, 0); 2350 } 2351 #endif /* PTL1_PANIC_DEBUG */ 2352 2353 2354 /* 2355 * Add to a memory list. 2356 * start = start of new memory segment 2357 * len = length of new memory segment in bytes 2358 * memlistp = pointer to array of available memory segment structures 2359 * curmemlistp = memory list to which to add segment. 2360 */ 2361 static void 2362 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2363 struct memlist **curmemlistp) 2364 { 2365 struct memlist *new; 2366 2367 new = *memlistp; 2368 new->address = start; 2369 new->size = len; 2370 *memlistp = new + 1; 2371 2372 memlist_insert(new, curmemlistp); 2373 } 2374 2375 /* 2376 * In the case of architectures that support dynamic addition of 2377 * memory at run-time there are two cases where memsegs need to 2378 * be initialized and added to the memseg list. 2379 * 1) memsegs that are constructed at startup. 2380 * 2) memsegs that are constructed at run-time on 2381 * hot-plug capable architectures. 2382 * This code was originally part of the function kphysm_init(). 2383 */ 2384 2385 static void 2386 memseg_list_add(struct memseg *memsegp) 2387 { 2388 struct memseg **prev_memsegp; 2389 pgcnt_t num; 2390 2391 /* insert in memseg list, decreasing number of pages order */ 2392 2393 num = MSEG_NPAGES(memsegp); 2394 2395 for (prev_memsegp = &memsegs; *prev_memsegp; 2396 prev_memsegp = &((*prev_memsegp)->next)) { 2397 if (num > MSEG_NPAGES(*prev_memsegp)) 2398 break; 2399 } 2400 2401 memsegp->next = *prev_memsegp; 2402 *prev_memsegp = memsegp; 2403 2404 if (kpm_enable) { 2405 memsegp->nextpa = (memsegp->next) ? 2406 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2407 2408 if (prev_memsegp != &memsegs) { 2409 struct memseg *msp; 2410 msp = (struct memseg *)((caddr_t)prev_memsegp - 2411 offsetof(struct memseg, next)); 2412 msp->nextpa = va_to_pa(memsegp); 2413 } else { 2414 memsegspa = va_to_pa(memsegs); 2415 } 2416 } 2417 } 2418 2419 /* 2420 * PSM add_physmem_cb(). US-II and newer processors have some 2421 * flavor of the prefetch capability implemented. We exploit 2422 * this capability for optimum performance. 2423 */ 2424 #define PREFETCH_BYTES 64 2425 2426 void 2427 add_physmem_cb(page_t *pp, pfn_t pnum) 2428 { 2429 extern void prefetch_page_w(void *); 2430 2431 pp->p_pagenum = pnum; 2432 2433 /* 2434 * Prefetch one more page_t into E$. To prevent future 2435 * mishaps with the sizeof(page_t) changing on us, we 2436 * catch this on debug kernels if we can't bring in the 2437 * entire hpage with 2 PREFETCH_BYTES reads. See 2438 * also, sun4u/cpu/cpu_module.c 2439 */ 2440 /*LINTED*/ 2441 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2442 prefetch_page_w((char *)pp); 2443 } 2444 2445 /* 2446 * kphysm_init() tackles the problem of initializing physical memory. 2447 * The old startup made some assumptions about the kernel living in 2448 * physically contiguous space which is no longer valid. 2449 */ 2450 static void 2451 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2452 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2453 { 2454 struct memlist *pmem; 2455 struct memseg *msp; 2456 pfn_t base; 2457 pgcnt_t num; 2458 pfn_t lastseg_pages_end = 0; 2459 pgcnt_t nelem_used = 0; 2460 2461 ASSERT(page_hash != NULL && page_hashsz != 0); 2462 2463 msp = memsegp; 2464 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2465 2466 /* 2467 * Build the memsegs entry 2468 */ 2469 num = btop(pmem->size); 2470 if (num > npages) 2471 num = npages; 2472 npages -= num; 2473 base = btop(pmem->address); 2474 2475 msp->pages = pp; 2476 msp->epages = pp + num; 2477 msp->pages_base = base; 2478 msp->pages_end = base + num; 2479 2480 if (kpm_enable) { 2481 pfn_t pbase_a; 2482 pfn_t pend_a; 2483 pfn_t prev_pend_a; 2484 pgcnt_t nelem; 2485 2486 msp->pagespa = va_to_pa(pp); 2487 msp->epagespa = va_to_pa(pp + num); 2488 pbase_a = kpmptop(ptokpmp(base)); 2489 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2490 nelem = ptokpmp(pend_a - pbase_a); 2491 msp->kpm_nkpmpgs = nelem; 2492 msp->kpm_pbase = pbase_a; 2493 if (lastseg_pages_end) { 2494 /* 2495 * Assume phys_avail is in ascending order 2496 * of physical addresses. 2497 */ 2498 ASSERT(base + num > lastseg_pages_end); 2499 prev_pend_a = kpmptop( 2500 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2501 2502 if (prev_pend_a > pbase_a) { 2503 /* 2504 * Overlap, more than one memseg may 2505 * point to the same kpm_page range. 2506 */ 2507 if (kpm_smallpages == 0) { 2508 msp->kpm_pages = 2509 (kpm_page_t *)kpm_pp - 1; 2510 kpm_pp = (uintptr_t) 2511 ((kpm_page_t *)kpm_pp 2512 + nelem - 1); 2513 } else { 2514 msp->kpm_spages = 2515 (kpm_spage_t *)kpm_pp - 1; 2516 kpm_pp = (uintptr_t) 2517 ((kpm_spage_t *)kpm_pp 2518 + nelem - 1); 2519 } 2520 nelem_used += nelem - 1; 2521 2522 } else { 2523 if (kpm_smallpages == 0) { 2524 msp->kpm_pages = 2525 (kpm_page_t *)kpm_pp; 2526 kpm_pp = (uintptr_t) 2527 ((kpm_page_t *)kpm_pp 2528 + nelem); 2529 } else { 2530 msp->kpm_spages = 2531 (kpm_spage_t *)kpm_pp; 2532 kpm_pp = (uintptr_t) 2533 ((kpm_spage_t *) 2534 kpm_pp + nelem); 2535 } 2536 nelem_used += nelem; 2537 } 2538 2539 } else { 2540 if (kpm_smallpages == 0) { 2541 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2542 kpm_pp = (uintptr_t) 2543 ((kpm_page_t *)kpm_pp + nelem); 2544 } else { 2545 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2546 kpm_pp = (uintptr_t) 2547 ((kpm_spage_t *)kpm_pp + nelem); 2548 } 2549 nelem_used = nelem; 2550 } 2551 2552 if (nelem_used > kpm_npages) 2553 panic("kphysm_init: kpm_pp overflow\n"); 2554 2555 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2556 lastseg_pages_end = msp->pages_end; 2557 } 2558 2559 memseg_list_add(msp); 2560 2561 /* 2562 * add_physmem() initializes the PSM part of the page 2563 * struct by calling the PSM back with add_physmem_cb(). 2564 * In addition it coalesces pages into larger pages as 2565 * it initializes them. 2566 */ 2567 add_physmem(pp, num, base); 2568 pp += num; 2569 msp++; 2570 } 2571 2572 build_pfn_hash(); 2573 } 2574 2575 /* 2576 * Kernel VM initialization. 2577 * Assumptions about kernel address space ordering: 2578 * (1) gap (user space) 2579 * (2) kernel text 2580 * (3) kernel data/bss 2581 * (4) gap 2582 * (5) kernel data structures 2583 * (6) gap 2584 * (7) debugger (optional) 2585 * (8) monitor 2586 * (9) gap (possibly null) 2587 * (10) dvma 2588 * (11) devices 2589 */ 2590 static void 2591 kvm_init(void) 2592 { 2593 /* 2594 * Put the kernel segments in kernel address space. 2595 */ 2596 rw_enter(&kas.a_lock, RW_WRITER); 2597 as_avlinit(&kas); 2598 2599 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2600 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2601 (void) segkmem_create(&ktextseg); 2602 2603 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2604 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2605 (void) segkmem_create(&ktexthole); 2606 2607 (void) seg_attach(&kas, (caddr_t)valloc_base, 2608 (size_t)(econtig32 - valloc_base), &kvalloc); 2609 (void) segkmem_create(&kvalloc); 2610 2611 if (kmem64_base) { 2612 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2613 (size_t)(kmem64_end - kmem64_base), &kmem64); 2614 (void) segkmem_create(&kmem64); 2615 } 2616 2617 /* 2618 * We're about to map out /boot. This is the beginning of the 2619 * system resource management transition. We can no longer 2620 * call into /boot for I/O or memory allocations. 2621 */ 2622 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2623 (void) segkmem_create(&kvseg); 2624 hblk_alloc_dynamic = 1; 2625 2626 /* 2627 * we need to preallocate pages for DR operations before enabling large 2628 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2629 */ 2630 memseg_remap_init(); 2631 2632 /* at this point we are ready to use large page heap */ 2633 segkmem_heap_lp_init(); 2634 2635 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2636 &kvseg32); 2637 (void) segkmem_create(&kvseg32); 2638 2639 /* 2640 * Create a segment for the debugger. 2641 */ 2642 (void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE, 2643 &kdebugseg); 2644 (void) segkmem_create(&kdebugseg); 2645 2646 rw_exit(&kas.a_lock); 2647 } 2648 2649 char obp_tte_str[] = 2650 "h# %x constant MMU_PAGESHIFT " 2651 "h# %x constant TTE8K " 2652 "h# %x constant SFHME_SIZE " 2653 "h# %x constant SFHME_TTE " 2654 "h# %x constant HMEBLK_TAG " 2655 "h# %x constant HMEBLK_NEXT " 2656 "h# %x constant HMEBLK_MISC " 2657 "h# %x constant HMEBLK_HME1 " 2658 "h# %x constant NHMENTS " 2659 "h# %x constant HBLK_SZMASK " 2660 "h# %x constant HBLK_RANGE_SHIFT " 2661 "h# %x constant HMEBP_HBLK " 2662 "h# %x constant HMEBUCKET_SIZE " 2663 "h# %x constant HTAG_SFMMUPSZ " 2664 "h# %x constant HTAG_REHASHSZ " 2665 "h# %x constant mmu_hashcnt " 2666 "h# %p constant uhme_hash " 2667 "h# %p constant khme_hash " 2668 "h# %x constant UHMEHASH_SZ " 2669 "h# %x constant KHMEHASH_SZ " 2670 "h# %p constant KHATID " 2671 "h# %x constant CTX_SIZE " 2672 "h# %x constant CTX_SFMMU " 2673 "h# %p constant ctxs " 2674 "h# %x constant ASI_MEM " 2675 2676 ": PHYS-X@ ( phys -- data ) " 2677 " ASI_MEM spacex@ " 2678 "; " 2679 2680 ": PHYS-W@ ( phys -- data ) " 2681 " ASI_MEM spacew@ " 2682 "; " 2683 2684 ": PHYS-L@ ( phys -- data ) " 2685 " ASI_MEM spaceL@ " 2686 "; " 2687 2688 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2689 " 3 * MMU_PAGESHIFT + " 2690 "; " 2691 2692 ": TTE_IS_VALID ( ttep -- flag ) " 2693 " PHYS-X@ 0< " 2694 "; " 2695 2696 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2697 " dup TTE8K = if " 2698 " drop HBLK_RANGE_SHIFT " 2699 " else " 2700 " TTE_PAGE_SHIFT " 2701 " then " 2702 "; " 2703 2704 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2705 " tuck >> swap MMU_PAGESHIFT - << " 2706 "; " 2707 2708 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2709 " >> over xor swap ( hash sfmmup ) " 2710 " KHATID <> if ( hash ) " 2711 " UHMEHASH_SZ and ( bucket ) " 2712 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2713 " else ( hash ) " 2714 " KHMEHASH_SZ and ( bucket ) " 2715 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2716 " then ( hmebp ) " 2717 "; " 2718 2719 ": HME_HASH_TABLE_SEARCH " 2720 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2721 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2722 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2723 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2724 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2725 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2726 " else " 2727 " hmeblk_next + phys-x@ false " 2728 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2729 " then " 2730 " else " 2731 " hmeblk_next + phys-x@ false " 2732 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2733 " then " 2734 " else " 2735 " true " 2736 " then " 2737 " until r> drop " 2738 "; " 2739 2740 ": CNUM_TO_SFMMUP ( cnum -- sfmmup ) " 2741 " CTX_SIZE * ctxs + CTX_SFMMU + " 2742 "x@ " 2743 "; " 2744 2745 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2746 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2747 " HTAG_REHASHSZ << or nip ( hblktag ) " 2748 "; " 2749 2750 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2751 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2752 " TTE8K = if ( hmeblkp addr ) " 2753 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2754 " else ( hmeblkp addr ) " 2755 " drop 0 ( hmeblkp 0 ) " 2756 " then ( hmeblkp hme-index ) " 2757 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2758 " SFHME_TTE + ( ttep ) " 2759 "; " 2760 2761 ": unix-tte ( addr cnum -- false | tte-data true ) " 2762 " CNUM_TO_SFMMUP ( addr sfmmup ) " 2763 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2764 " 2dup swap i HME_HASH_SHIFT " 2765 "( addr sfmmup sfmmup addr hmeshift ) " 2766 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2767 " over i 4 pick " 2768 "( addr sfmmup hmebp sfmmup rehash addr ) " 2769 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2770 " HME_HASH_TABLE_SEARCH " 2771 "( addr sfmmup { null | hmeblkp } ) " 2772 " ?dup if ( addr sfmmup hmeblkp ) " 2773 " nip swap HBLK_TO_TTEP ( ttep ) " 2774 " dup TTE_IS_VALID if ( valid-ttep ) " 2775 " PHYS-X@ true ( tte-data true ) " 2776 " else ( invalid-tte ) " 2777 " drop false ( false ) " 2778 " then ( false | tte-data true ) " 2779 " unloop exit ( false | tte-data true ) " 2780 " then ( addr sfmmup ) " 2781 " loop ( addr sfmmup ) " 2782 " 2drop false ( false ) " 2783 "; " 2784 ; 2785 2786 void 2787 create_va_to_tte(void) 2788 { 2789 char *bp; 2790 extern int khmehash_num, uhmehash_num; 2791 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2792 2793 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2794 2795 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2796 2797 /* 2798 * Teach obp how to parse our sw ttes. 2799 */ 2800 (void) sprintf(bp, obp_tte_str, 2801 MMU_PAGESHIFT, 2802 TTE8K, 2803 sizeof (struct sf_hment), 2804 OFFSET(struct sf_hment, hme_tte), 2805 OFFSET(struct hme_blk, hblk_tag), 2806 OFFSET(struct hme_blk, hblk_nextpa), 2807 OFFSET(struct hme_blk, hblk_misc), 2808 OFFSET(struct hme_blk, hblk_hme), 2809 NHMENTS, 2810 HBLK_SZMASK, 2811 HBLK_RANGE_SHIFT, 2812 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2813 sizeof (struct hmehash_bucket), 2814 HTAG_SFMMUPSZ, 2815 HTAG_REHASHSZ, 2816 mmu_hashcnt, 2817 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2818 (caddr_t)va_to_pa((caddr_t)khme_hash), 2819 UHMEHASH_SZ, 2820 KHMEHASH_SZ, 2821 KHATID, 2822 sizeof (struct ctx), 2823 OFFSET(struct ctx, ctx_sfmmu), 2824 ctxs, 2825 ASI_MEM); 2826 prom_interpret(bp, 0, 0, 0, 0, 0); 2827 2828 kobj_free(bp, MMU_PAGESIZE); 2829 } 2830 2831 void 2832 install_va_to_tte(void) 2833 { 2834 /* 2835 * advise prom that he can use unix-tte 2836 */ 2837 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2838 } 2839 2840 2841 static char *create_node = 2842 "root-device " 2843 "new-device " 2844 "\" os-io\" device-name " 2845 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2846 " 2>r swap 2 2r> ['] $callback catch if " 2847 " 2drop 3drop 0 " 2848 " then " 2849 "; " 2850 ": read ( adr,len -- #read ) " 2851 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2852 " ( retN ... ret1 N ) " 2853 " ?dup if " 2854 " swap >r 1- 0 ?do drop loop r> " 2855 " else " 2856 " -2 " 2857 " then l->n " 2858 "; " 2859 ": write ( adr,len -- #written ) " 2860 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2861 " ( retN ... ret1 N ) " 2862 " ?dup if " 2863 " swap >r 1- 0 ?do drop loop r> " 2864 " else " 2865 " 0 " 2866 " then " 2867 "; " 2868 ": poll-tty ( -- ) ; " 2869 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2870 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2871 ": cb-give/take ( $method -- ) " 2872 " 0 -rot ['] $callback catch ?dup if " 2873 " >r 2drop 2drop r> throw " 2874 " else " 2875 " 0 ?do drop loop " 2876 " then " 2877 "; " 2878 ": give ( -- ) \" exit-input\" cb-give/take ; " 2879 ": take ( -- ) \" enter-input\" cb-give/take ; " 2880 ": open ( -- ok? ) true ; " 2881 ": close ( -- ) ; " 2882 "finish-device " 2883 "device-end "; 2884 2885 /* 2886 * Create the obp input/output node only if the USB keyboard is the 2887 * standard input device. When the USB software takes over the 2888 * input device at the time consconfig runs, it will switch OBP's 2889 * notion of the input device to this node. Whenever the 2890 * forth user interface is used after this switch, the node will 2891 * call back into the kernel for console input. 2892 * 2893 * This callback mechanism is currently only used when the USB keyboard 2894 * is the input device. If a serial device such as ttya or 2895 * a UART with a Type 5 keyboard attached is used, obp takes over the 2896 * serial device when the system goes to the debugger after the system is 2897 * booted. This sharing of the relatively simple serial device is difficult 2898 * but possible. Sharing the USB host controller is impossible due 2899 * its complexity 2900 */ 2901 static void 2902 startup_create_input_node(void) 2903 { 2904 char *stdin_path; 2905 2906 /* 2907 * If usb_node_debug is set in /etc/system 2908 * then the user would like to test the callbacks 2909 * from the input node regardless of whether or 2910 * not the USB keyboard is the console input. 2911 * This variable is useful for debugging. 2912 */ 2913 if (usb_node_debug) { 2914 2915 prom_interpret(create_node, 0, 0, 0, 0, 0); 2916 2917 return; 2918 } 2919 2920 /* Obtain the console input device */ 2921 stdin_path = prom_stdinpath(); 2922 2923 /* 2924 * If the string "usb" and "keyboard" are in the path 2925 * then a USB keyboard is the console input device, 2926 * create the node. 2927 */ 2928 if ((strstr(stdin_path, "usb") != 0) && 2929 (strstr(stdin_path, "keyboard") != 0)) { 2930 2931 prom_interpret(create_node, 0, 0, 0, 0, 0); 2932 } 2933 } 2934 2935 2936 static void 2937 do_prom_version_check(void) 2938 { 2939 int i; 2940 pnode_t node; 2941 char buf[64]; 2942 static char drev[] = "Down-rev firmware detected%s\n" 2943 "\tPlease upgrade to the following minimum version:\n" 2944 "\t\t%s\n"; 2945 2946 i = prom_version_check(buf, sizeof (buf), &node); 2947 2948 if (i == PROM_VER64_OK) 2949 return; 2950 2951 if (i == PROM_VER64_UPGRADE) { 2952 cmn_err(CE_WARN, drev, "", buf); 2953 2954 #ifdef DEBUG 2955 prom_enter_mon(); /* Type 'go' to continue */ 2956 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2957 return; 2958 #else 2959 halt(0); 2960 #endif 2961 } 2962 2963 /* 2964 * The other possibility is that this is a server running 2965 * good firmware, but down-rev firmware was detected on at 2966 * least one other cpu board. We just complain if we see 2967 * that. 2968 */ 2969 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2970 } 2971 2972 static void 2973 kpm_init() 2974 { 2975 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 2976 kpm_pgsz = 1ull << kpm_pgshft; 2977 kpm_pgoff = kpm_pgsz - 1; 2978 kpmp2pshft = kpm_pgshft - PAGESHIFT; 2979 kpmpnpgs = 1 << kpmp2pshft; 2980 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 2981 } 2982 2983 void 2984 kpm_npages_setup(int memblocks) 2985 { 2986 /* 2987 * npages can be scattered in a maximum of 'memblocks' 2988 */ 2989 kpm_npages = ptokpmpr(npages) + memblocks; 2990 } 2991 2992 /* 2993 * Must be defined in platform dependent code. 2994 */ 2995 extern caddr_t modtext; 2996 extern size_t modtext_sz; 2997 extern caddr_t moddata; 2998 2999 #define HEAPTEXT_ARENA(addr) \ 3000 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3001 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3002 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3003 3004 #define HEAPTEXT_OVERSIZED(addr) \ 3005 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3006 3007 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3008 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3009 kmutex_t texthole_lock; 3010 3011 char kern_bootargs[OBP_MAXPATHLEN]; 3012 3013 void 3014 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3015 { 3016 uintptr_t addr, limit; 3017 3018 addr = HEAPTEXT_BASE; 3019 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3020 3021 /* 3022 * Before we initialize the text_arena, we want to punch holes in the 3023 * underlying heaptext_arena. This guarantees that for any text 3024 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3025 */ 3026 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3027 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3028 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3029 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3030 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3031 } 3032 3033 /* 3034 * Allocate one page at the oversize to break up the text region 3035 * from the oversized region. 3036 */ 3037 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3038 (void *)limit, (void *)(limit + PAGESIZE), 3039 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3040 3041 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3042 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3043 heaptext_arena, 0, VM_SLEEP); 3044 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3045 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3046 } 3047 3048 caddr_t 3049 kobj_text_alloc(vmem_t *arena, size_t size) 3050 { 3051 caddr_t rval, better; 3052 3053 /* 3054 * First, try a sleeping allocation. 3055 */ 3056 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3057 3058 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3059 return (rval); 3060 3061 /* 3062 * We didn't get the area that we wanted. We're going to try to do an 3063 * allocation with explicit constraints. 3064 */ 3065 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3066 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3067 VM_NOSLEEP | VM_BESTFIT); 3068 3069 if (better != NULL) { 3070 /* 3071 * That worked. Free our first attempt and return. 3072 */ 3073 vmem_free(arena, rval, size); 3074 return (better); 3075 } 3076 3077 /* 3078 * That didn't work; we'll have to return our first attempt. 3079 */ 3080 return (rval); 3081 } 3082 3083 caddr_t 3084 kobj_texthole_alloc(caddr_t addr, size_t size) 3085 { 3086 int arena = HEAPTEXT_ARENA(addr); 3087 char c[30]; 3088 uintptr_t base; 3089 3090 if (HEAPTEXT_OVERSIZED(addr)) { 3091 /* 3092 * If this is an oversized allocation, there is no text hole 3093 * available for it; return NULL. 3094 */ 3095 return (NULL); 3096 } 3097 3098 mutex_enter(&texthole_lock); 3099 3100 if (texthole_arena[arena] == NULL) { 3101 ASSERT(texthole_source[arena] == NULL); 3102 3103 if (arena == 0) { 3104 texthole_source[0] = vmem_create("module_text_holesrc", 3105 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3106 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3107 0, VM_SLEEP); 3108 } else { 3109 base = HEAPTEXT_BASE + 3110 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3111 3112 (void) snprintf(c, sizeof (c), 3113 "heaptext_holesrc_%d", arena); 3114 3115 texthole_source[arena] = vmem_create(c, (void *)base, 3116 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3117 0, VM_SLEEP); 3118 } 3119 3120 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3121 3122 texthole_arena[arena] = vmem_create(c, NULL, 0, 3123 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3124 texthole_source[arena], 0, VM_SLEEP); 3125 } 3126 3127 mutex_exit(&texthole_lock); 3128 3129 ASSERT(texthole_arena[arena] != NULL); 3130 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3131 return (vmem_alloc(texthole_arena[arena], size, 3132 VM_BESTFIT | VM_NOSLEEP)); 3133 } 3134 3135 void 3136 kobj_texthole_free(caddr_t addr, size_t size) 3137 { 3138 int arena = HEAPTEXT_ARENA(addr); 3139 3140 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3141 ASSERT(texthole_arena[arena] != NULL); 3142 vmem_free(texthole_arena[arena], addr, size); 3143 } 3144