1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern int cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 74 extern void mach_dump_buffer_init(void); 75 extern void mach_descrip_init(void); 76 extern void mach_descrip_startup_fini(void); 77 extern void mach_memscrub(void); 78 extern void mach_fpras(void); 79 extern void mach_cpu_halt_idle(void); 80 extern void mach_hw_copy_limit(void); 81 extern void load_mach_drivers(void); 82 extern void load_tod_module(void); 83 #pragma weak load_tod_module 84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata); 86 #pragma weak ndata_alloc_mmfsa 87 88 extern void cif_init(void); 89 #pragma weak cif_init 90 91 extern void parse_idprom(void); 92 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 93 extern void mem_config_init(void); 94 extern void memseg_remap_init(void); 95 96 extern void mach_kpm_init(void); 97 extern int size_pse_array(pgcnt_t, int); 98 99 /* 100 * External Data: 101 */ 102 extern int vac_size; /* cache size in bytes */ 103 extern uint_t vac_mask; /* VAC alignment consistency mask */ 104 extern uint_t vac_colors; 105 106 /* 107 * Global Data Definitions: 108 */ 109 110 /* 111 * XXX - Don't port this to new architectures 112 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 113 * 'romp' has no use with a prom with an IEEE 1275 client interface. 114 * The driver doesn't use the value, but it depends on the symbol. 115 */ 116 void *romp; /* veritas driver won't load without romp 4154976 */ 117 /* 118 * Declare these as initialized data so we can patch them. 119 */ 120 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 121 pgcnt_t segkpsize = 122 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 123 uint_t segmap_percent = 12; /* Size of segmap segment */ 124 125 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 126 int vac_copyback = 1; 127 char *cache_mode = NULL; 128 int use_mix = 1; 129 int prom_debug = 0; 130 131 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 132 uint_t tba_taken_over = 0; 133 134 caddr_t s_text; /* start of kernel text segment */ 135 caddr_t e_text; /* end of kernel text segment */ 136 caddr_t s_data; /* start of kernel data segment */ 137 caddr_t e_data; /* end of kernel data segment */ 138 139 caddr_t modtext; /* beginning of module text */ 140 size_t modtext_sz; /* size of module text */ 141 caddr_t moddata; /* beginning of module data reserve */ 142 caddr_t e_moddata; /* end of module data reserve */ 143 144 /* 145 * End of first block of contiguous kernel in 32-bit virtual address space 146 */ 147 caddr_t econtig32; /* end of first blk of contiguous kernel */ 148 149 caddr_t ncbase; /* beginning of non-cached segment */ 150 caddr_t ncend; /* end of non-cached segment */ 151 152 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 153 caddr_t nalloc_base; /* beginning of nucleus allocation */ 154 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 155 caddr_t valloc_base; /* beginning of kvalloc segment */ 156 157 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 158 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 159 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 160 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 161 int kmem64_szc; /* page size code */ 162 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 163 164 uintptr_t shm_alignment; /* VAC address consistency modulus */ 165 struct memlist *phys_install; /* Total installed physical memory */ 166 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 167 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 168 struct memlist *nopp_list; /* pages with no backing page structs */ 169 struct memlist ndata; /* memlist of nucleus allocatable memory */ 170 int memexp_flag; /* memory expansion card flag */ 171 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 172 pgcnt_t obp_pages; /* Physical pages used by OBP */ 173 174 /* 175 * VM data structures 176 */ 177 long page_hashsz; /* Size of page hash table (power of two) */ 178 struct page *pp_base; /* Base of system page struct array */ 179 size_t pp_sz; /* Size in bytes of page struct array */ 180 struct page **page_hash; /* Page hash table */ 181 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 182 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 183 int pse_shift; /* log2(pse_table_size) */ 184 struct seg ktextseg; /* Segment used for kernel executable image */ 185 struct seg kvalloc; /* Segment used for "valloc" mapping */ 186 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 187 struct seg ktexthole; /* Segment used for nucleus text hole */ 188 struct seg kmapseg; /* Segment used for generic kernel mappings */ 189 struct seg kpmseg; /* Segment used for physical mapping */ 190 struct seg kdebugseg; /* Segment used for the kernel debugger */ 191 192 void *kpm_pp_base; /* Base of system kpm_page array */ 193 size_t kpm_pp_sz; /* Size of system kpm_page array */ 194 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 195 196 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 197 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 198 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 199 200 int segzio_fromheap = 0; /* zio allocations occur from heap */ 201 caddr_t segzio_base; /* Base address of segzio */ 202 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 203 204 /* 205 * debugger pages (if allocated) 206 */ 207 struct vnode kdebugvp; 208 209 /* 210 * VA range available to the debugger 211 */ 212 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 213 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 214 215 /* 216 * Segment for relocated kernel structures in 64-bit large RAM kernels 217 */ 218 struct seg kmem64; 219 220 struct memseg *memseg_free; 221 222 struct vnode unused_pages_vp; 223 224 /* 225 * VM data structures allocated early during boot. 226 */ 227 size_t pagehash_sz; 228 uint64_t memlist_sz; 229 230 char tbr_wr_addr_inited = 0; 231 232 caddr_t mpo_heap32_buf = NULL; 233 size_t mpo_heap32_bufsz = 0; 234 235 /* 236 * Static Routines: 237 */ 238 static int ndata_alloc_memseg(struct memlist *, size_t); 239 static void memlist_new(uint64_t, uint64_t, struct memlist **); 240 static void memlist_add(uint64_t, uint64_t, 241 struct memlist **, struct memlist **); 242 static void kphysm_init(void); 243 static void kvm_init(void); 244 static void install_kmem64_tte(void); 245 246 static void startup_init(void); 247 static void startup_memlist(void); 248 static void startup_modules(void); 249 static void startup_bop_gone(void); 250 static void startup_vm(void); 251 static void startup_end(void); 252 static void setup_cage_params(void); 253 static void startup_create_io_node(void); 254 255 static pgcnt_t npages; 256 static struct memlist *memlist; 257 void *memlist_end; 258 259 static pgcnt_t bop_alloc_pages; 260 static caddr_t hblk_base; 261 uint_t hblk_alloc_dynamic = 0; 262 uint_t hblk1_min = H1MIN; 263 264 265 /* 266 * Hooks for unsupported platforms and down-rev firmware 267 */ 268 int iam_positron(void); 269 #pragma weak iam_positron 270 static void do_prom_version_check(void); 271 272 /* 273 * After receiving a thermal interrupt, this is the number of seconds 274 * to delay before shutting off the system, assuming 275 * shutdown fails. Use /etc/system to change the delay if this isn't 276 * large enough. 277 */ 278 int thermal_powerdown_delay = 1200; 279 280 /* 281 * Used to hold off page relocations into the cage until OBP has completed 282 * its boot-time handoff of its resources to the kernel. 283 */ 284 int page_relocate_ready = 0; 285 286 /* 287 * Enable some debugging messages concerning memory usage... 288 */ 289 #ifdef DEBUGGING_MEM 290 static int debugging_mem; 291 static void 292 printmemlist(char *title, struct memlist *list) 293 { 294 if (!debugging_mem) 295 return; 296 297 printf("%s\n", title); 298 299 while (list) { 300 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 301 (uint32_t)(list->address >> 32), (uint32_t)list->address, 302 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 303 list = list->next; 304 } 305 } 306 307 void 308 printmemseg(struct memseg *memseg) 309 { 310 if (!debugging_mem) 311 return; 312 313 printf("memseg\n"); 314 315 while (memseg) { 316 prom_printf("\tpage = 0x%p, epage = 0x%p, " 317 "pfn = 0x%x, epfn = 0x%x\n", 318 memseg->pages, memseg->epages, 319 memseg->pages_base, memseg->pages_end); 320 memseg = memseg->next; 321 } 322 } 323 324 #define debug_pause(str) halt((str)) 325 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 326 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 327 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 328 #define MPRINTF3(str, a, b, c) \ 329 if (debugging_mem) prom_printf((str), (a), (b), (c)) 330 #else /* DEBUGGING_MEM */ 331 #define MPRINTF(str) 332 #define MPRINTF1(str, a) 333 #define MPRINTF2(str, a, b) 334 #define MPRINTF3(str, a, b, c) 335 #endif /* DEBUGGING_MEM */ 336 337 338 /* 339 * 340 * Kernel's Virtual Memory Layout. 341 * /-----------------------\ 342 * 0xFFFFFFFF.FFFFFFFF -| |- 343 * | OBP's virtual page | 344 * | tables | 345 * 0xFFFFFFFC.00000000 -|-----------------------|- 346 * : : 347 * : : 348 * -|-----------------------|- 349 * | segzio | (base and size vary) 350 * 0xFFFFFE00.00000000 -|-----------------------|- 351 * | | Ultrasparc I/II support 352 * | segkpm segment | up to 2TB of physical 353 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 354 * | | 355 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 356 * : : 357 * : : 358 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 359 * | | ^ 360 * | UltraSPARC I/II call | | 361 * | bug requires an extra | | 362 * | 4 GB of space between | | 363 * | hole and used RAM | | 364 * | | | 365 * 0xFFFFF800.00000000 -|-----------------------|- | 366 * | | | 367 * | Virtual Address Hole | UltraSPARC 368 * | on UltraSPARC I/II | I/II * ONLY * 369 * | | | 370 * 0x00000800.00000000 -|-----------------------|- | 371 * | | | 372 * | UltraSPARC I/II call | | 373 * | bug requires an extra | | 374 * | 4 GB of space between | | 375 * | hole and used RAM | | 376 * | | v 377 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 378 * : : ^ 379 * : : | 380 * |-----------------------| | 381 * | | | 382 * | ecache flush area | | 383 * | (twice largest e$) | | 384 * | | | 385 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 386 * | overmapped area | alignend_end | 387 * | (kmem64_alignsize | | 388 * | boundary) | | 389 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 390 * | | | 391 * | 64-bit kernel ONLY | | 392 * | | | 393 * | kmem64 segment | | 394 * | | | 395 * | (Relocated extra HME | Approximately 396 * | block allocations, | 1 TB of virtual 397 * | memnode freelists, | address space 398 * | HME hash buckets, | | 399 * | mml_table, kpmp_table,| | 400 * | page_t array and | | 401 * | hashblock pool to | | 402 * | avoid hard-coded | | 403 * | 32-bit vaddr | | 404 * | limitations) | | 405 * | | v 406 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 407 * | | 408 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 409 * | | 410 * 0x00000300.00000000 -|-----------------------|- SYSBASE 411 * : : 412 * : : 413 * -|-----------------------|- 414 * | | 415 * | segmap segment | SEGMAPSIZE (1/8th physmem, 416 * | | 256G MAX) 417 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 418 * : : 419 * : : 420 * -|-----------------------|- 421 * | | 422 * | segkp | SEGKPSIZE (2GB) 423 * | | 424 * | | 425 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 426 * | | 427 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 428 * | | (SEGKPBASE - 0x400000) 429 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 430 * | | (MEMSCRUBBASE - NCARGS) 431 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 432 * | | (ARGSBASE - PPMAPSIZE) 433 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 434 * | | 435 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 436 * | | 437 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 438 * : : 439 * : : 440 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 441 * | | 442 * | OBP | 443 * | | 444 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 445 * | kmdb | 446 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 447 * : : 448 * : : 449 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 450 * | | 451 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 452 * | | ~64MB) 453 * 0x00000000.70002000 -|-----------------------| 454 * | panicbuf | 455 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 456 * | boot-time | 457 * | temporary space | 458 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 459 * : : 460 * : : 461 * | | 462 * |-----------------------|- econtig32 463 * | vm structures | 464 * 0x00000000.01C00000 |-----------------------|- nalloc_end 465 * | TSBs | 466 * |-----------------------|- end/nalloc_base 467 * | kernel data & bss | 468 * 0x00000000.01800000 -|-----------------------| 469 * : nucleus text hole : 470 * 0x00000000.01400000 -|-----------------------| 471 * : : 472 * |-----------------------| 473 * | module text | 474 * |-----------------------|- e_text/modtext 475 * | kernel text | 476 * |-----------------------| 477 * | trap table (48k) | 478 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 479 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 480 * |-----------------------| 481 * | | 482 * | invalid | 483 * | | 484 * 0x00000000.00000000 _|_______________________| 485 * 486 * 487 * 488 * 32-bit User Virtual Memory Layout. 489 * /-----------------------\ 490 * | | 491 * | invalid | 492 * | | 493 * 0xFFC00000 -|-----------------------|- USERLIMIT 494 * | user stack | 495 * : : 496 * : : 497 * : : 498 * | user data | 499 * -|-----------------------|- 500 * | user text | 501 * 0x00002000 -|-----------------------|- 502 * | invalid | 503 * 0x00000000 _|_______________________| 504 * 505 * 506 * 507 * 64-bit User Virtual Memory Layout. 508 * /-----------------------\ 509 * | | 510 * | invalid | 511 * | | 512 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 513 * | user stack | 514 * : : 515 * : : 516 * : : 517 * | user data | 518 * -|-----------------------|- 519 * | user text | 520 * 0x00000000.01000000 -|-----------------------|- 521 * | invalid | 522 * 0x00000000.00000000 _|_______________________| 523 */ 524 525 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 526 extern uint64_t ecache_flush_address(void); 527 528 #pragma weak load_platform_modules 529 #pragma weak plat_startup_memlist 530 #pragma weak ecache_init_scrub_flush_area 531 #pragma weak ecache_flush_address 532 533 534 /* 535 * By default the DR Cage is enabled for maximum OS 536 * MPSS performance. Users needing to disable the cage mechanism 537 * can set this variable to zero via /etc/system. 538 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 539 * will result in loss of DR functionality. 540 * Platforms wishing to disable kernel Cage by default 541 * should do so in their set_platform_defaults() routine. 542 */ 543 int kernel_cage_enable = 1; 544 545 static void 546 setup_cage_params(void) 547 { 548 void (*func)(void); 549 550 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 551 if (func != NULL) { 552 (*func)(); 553 return; 554 } 555 556 if (kernel_cage_enable == 0) { 557 return; 558 } 559 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 560 561 if (kcage_on) { 562 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 563 } else { 564 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 565 } 566 567 } 568 569 /* 570 * Machine-dependent startup code 571 */ 572 void 573 startup(void) 574 { 575 startup_init(); 576 if (&startup_platform) 577 startup_platform(); 578 startup_memlist(); 579 startup_modules(); 580 setup_cage_params(); 581 startup_bop_gone(); 582 startup_vm(); 583 startup_end(); 584 } 585 586 struct regs sync_reg_buf; 587 uint64_t sync_tt; 588 589 void 590 sync_handler(void) 591 { 592 struct panic_trap_info ti; 593 int i; 594 595 /* 596 * Prevent trying to talk to the other CPUs since they are 597 * sitting in the prom and won't reply. 598 */ 599 for (i = 0; i < NCPU; i++) { 600 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 601 cpu[i]->cpu_flags &= ~CPU_READY; 602 cpu[i]->cpu_flags |= CPU_QUIESCED; 603 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 604 } 605 } 606 607 /* 608 * We've managed to get here without going through the 609 * normal panic code path. Try and save some useful 610 * information. 611 */ 612 if (!panicstr && (curthread->t_panic_trap == NULL)) { 613 ti.trap_type = sync_tt; 614 ti.trap_regs = &sync_reg_buf; 615 ti.trap_addr = NULL; 616 ti.trap_mmu_fsr = 0x0; 617 618 curthread->t_panic_trap = &ti; 619 } 620 621 /* 622 * If we're re-entering the panic path, update the signature 623 * block so that the SC knows we're in the second part of panic. 624 */ 625 if (panicstr) 626 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 627 628 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 629 panic("sync initiated"); 630 } 631 632 633 static void 634 startup_init(void) 635 { 636 /* 637 * We want to save the registers while we're still in OBP 638 * so that we know they haven't been fiddled with since. 639 * (In principle, OBP can't change them just because it 640 * makes a callback, but we'd rather not depend on that 641 * behavior.) 642 */ 643 char sync_str[] = 644 "warning @ warning off : sync " 645 "%%tl-c %%tstate h# %p x! " 646 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 647 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 648 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 649 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 650 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 651 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 652 "%%y h# %p l! %%tl-c %%tt h# %p x! " 653 "sync ; warning !"; 654 655 /* 656 * 20 == num of %p substrings 657 * 16 == max num of chars %p will expand to. 658 */ 659 char bp[sizeof (sync_str) + 16 * 20]; 660 661 /* 662 * Initialize ptl1 stack for the 1st CPU. 663 */ 664 ptl1_init_cpu(&cpu0); 665 666 /* 667 * Initialize the address map for cache consistent mappings 668 * to random pages; must be done after vac_size is set. 669 */ 670 ppmapinit(); 671 672 /* 673 * Initialize the PROM callback handler. 674 */ 675 init_vx_handler(); 676 677 /* 678 * have prom call sync_callback() to handle the sync and 679 * save some useful information which will be stored in the 680 * core file later. 681 */ 682 (void) sprintf((char *)bp, sync_str, 683 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 684 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 685 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 686 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 687 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 688 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 689 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 690 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 691 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 692 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 693 prom_interpret(bp, 0, 0, 0, 0, 0); 694 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 695 } 696 697 698 size_t 699 calc_pp_sz(pgcnt_t npages) 700 { 701 702 return (npages * sizeof (struct page)); 703 } 704 705 size_t 706 calc_kpmpp_sz(pgcnt_t npages) 707 { 708 709 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 710 kpm_pgsz = 1ull << kpm_pgshft; 711 kpm_pgoff = kpm_pgsz - 1; 712 kpmp2pshft = kpm_pgshft - PAGESHIFT; 713 kpmpnpgs = 1 << kpmp2pshft; 714 715 if (kpm_smallpages == 0) { 716 /* 717 * Avoid fragmentation problems in kphysm_init() 718 * by allocating for all of physical memory 719 */ 720 kpm_npages = ptokpmpr(physinstalled); 721 return (kpm_npages * sizeof (kpm_page_t)); 722 } else { 723 kpm_npages = npages; 724 return (kpm_npages * sizeof (kpm_spage_t)); 725 } 726 } 727 728 size_t 729 calc_pagehash_sz(pgcnt_t npages) 730 { 731 732 /* 733 * The page structure hash table size is a power of 2 734 * such that the average hash chain length is PAGE_HASHAVELEN. 735 */ 736 page_hashsz = npages / PAGE_HASHAVELEN; 737 page_hashsz = 1 << highbit(page_hashsz); 738 return (page_hashsz * sizeof (struct page *)); 739 } 740 741 void 742 alloc_kmem64(caddr_t base, caddr_t end) 743 { 744 int i; 745 caddr_t aligned_end = NULL; 746 747 /* 748 * Make one large memory alloc after figuring out the 64-bit size. This 749 * will enable use of the largest page size appropriate for the system 750 * architecture. 751 */ 752 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 753 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 754 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 755 size_t alloc_size, alignsize; 756 #if !defined(C_OBP) 757 unsigned long long pa; 758 #endif /* !C_OBP */ 759 760 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 761 continue; 762 alignsize = TTEBYTES(i); 763 kmem64_szc = i; 764 765 /* limit page size for small memory */ 766 if (mmu_btop(alignsize) > (npages >> 2)) 767 continue; 768 769 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 770 alloc_size = aligned_end - base; 771 #if !defined(C_OBP) 772 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 773 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 774 kmem64_pabase = pa; 775 kmem64_aligned_end = aligned_end; 776 install_kmem64_tte(); 777 break; 778 } else { 779 prom_free_phys(alloc_size, pa); 780 } 781 } 782 #else /* !C_OBP */ 783 if (prom_alloc(base, alloc_size, alignsize) == base) { 784 kmem64_pabase = va_to_pa(kmem64_base); 785 kmem64_aligned_end = aligned_end; 786 break; 787 } 788 #endif /* !C_OBP */ 789 if (i == TTE8K) { 790 prom_panic("kmem64 allocation failure"); 791 } 792 } 793 ASSERT(aligned_end != NULL); 794 } 795 796 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 797 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 798 799 #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 800 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 801 (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 802 803 #if !defined(C_OBP) 804 /* 805 * Install a temporary tte handler in OBP for kmem64 area. 806 * 807 * We map kmem64 area with large pages before the trap table is taken 808 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 809 * the same area. Duplicate tlb entries with different page sizes 810 * cause unpredicatble behavior. To avoid this, we don't create 811 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 812 * OBP). Instead, we manage translations with a temporary va>tte-data 813 * handler (kmem64-tte). This handler is replaced by unix-tte when 814 * the trap table is taken over. 815 * 816 * The temporary handler knows the physical address of the kmem64 817 * area. It uses the prom's pgmap@ Forth word for other addresses. 818 * 819 * We have to use BOP_ALLOC() method for C-OBP platforms because 820 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 821 * sun4u platforms. On sun4u we flush tlb after trap table is taken 822 * over if we use large pages for kernel heap and kmem64. Since sun4u 823 * prom (unlike sun4v) calls va>tte-data first for client address 824 * translation prom's ttes for kmem64 can't get into TLB even if we 825 * later switch to prom's trap table again. C-OBP uses 4M pages for 826 * client mappings when possible so on all platforms we get the 827 * benefit from large mappings for kmem64 area immediately during 828 * boot. 829 * 830 * pseudo code: 831 * if (context != 0) { 832 * return false 833 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 834 * tte = tte_template + 835 * (((miss_va & pagemask) - kmem64_base)); 836 * return tte, true 837 * } else { 838 * return pgmap@ result 839 * } 840 */ 841 char kmem64_obp_str[] = 842 "h# %lx constant kmem64-base " 843 "h# %lx constant kmem64-end " 844 "h# %lx constant kmem64-pagemask " 845 "h# %lx constant kmem64-template " 846 847 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 848 " if ( addr ) " 849 " drop false exit then ( false ) " 850 " dup kmem64-base kmem64-end within if ( addr ) " 851 " kmem64-pagemask and ( addr' ) " 852 " kmem64-base - ( addr' ) " 853 " kmem64-template + ( tte ) " 854 " true ( tte true ) " 855 " else ( addr ) " 856 " pgmap@ ( tte ) " 857 " dup 0< if true else drop false then ( tte true | false ) " 858 " then ( tte true | false ) " 859 "; " 860 861 "' kmem64-tte is va>tte-data " 862 ; 863 864 static void 865 install_kmem64_tte() 866 { 867 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 868 tte_t tte; 869 870 PRM_DEBUG(kmem64_pabase); 871 PRM_DEBUG(kmem64_szc); 872 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 873 PROC_DATA | HAT_NOSYNC, kmem64_szc); 874 PRM_DEBUG(tte.ll); 875 (void) sprintf(b, kmem64_obp_str, 876 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 877 ASSERT(strlen(b) < sizeof (b)); 878 prom_interpret(b, 0, 0, 0, 0, 0); 879 } 880 #endif /* !C_OBP */ 881 882 /* 883 * As OBP takes up some RAM when the system boots, pages will already be "lost" 884 * to the system and reflected in npages by the time we see it. 885 * 886 * We only want to allocate kernel structures in the 64-bit virtual address 887 * space on systems with enough RAM to make the overhead of keeping track of 888 * an extra kernel memory segment worthwhile. 889 * 890 * Since OBP has already performed its memory allocations by this point, if we 891 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 892 * memory in the 64-bit virtual address space; otherwise keep allocations 893 * contiguous with we've mapped so far in the 32-bit virtual address space. 894 */ 895 #define MINMOVE_RAM_MB ((size_t)1900) 896 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 897 898 pgcnt_t tune_npages = (pgcnt_t) 899 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 900 901 #pragma weak page_set_colorequiv_arr_cpu 902 extern void page_set_colorequiv_arr_cpu(void); 903 extern void page_set_colorequiv_arr(void); 904 905 906 static void 907 startup_memlist(void) 908 { 909 size_t hmehash_sz, pagelist_sz, tt_sz; 910 size_t psetable_sz; 911 caddr_t alloc_base; 912 caddr_t memspace; 913 struct memlist *cur; 914 size_t syslimit = (size_t)SYSLIMIT; 915 size_t sysbase = (size_t)SYSBASE; 916 917 /* 918 * Initialize enough of the system to allow kmem_alloc to work by 919 * calling boot to allocate its memory until the time that 920 * kvm_init is completed. The page structs are allocated after 921 * rounding up end to the nearest page boundary; the memsegs are 922 * initialized and the space they use comes from the kernel heap. 923 * With appropriate initialization, they can be reallocated later 924 * to a size appropriate for the machine's configuration. 925 * 926 * At this point, memory is allocated for things that will never 927 * need to be freed, this used to be "valloced". This allows a 928 * savings as the pages don't need page structures to describe 929 * them because them will not be managed by the vm system. 930 */ 931 932 /* 933 * We're loaded by boot with the following configuration (as 934 * specified in the sun4u/conf/Mapfile): 935 * 936 * text: 4 MB chunk aligned on a 4MB boundary 937 * data & bss: 4 MB chunk aligned on a 4MB boundary 938 * 939 * These two chunks will eventually be mapped by 2 locked 4MB 940 * ttes and will represent the nucleus of the kernel. This gives 941 * us some free space that is already allocated, some or all of 942 * which is made available to kernel module text. 943 * 944 * The free space in the data-bss chunk is used for nucleus 945 * allocatable data structures and we reserve it using the 946 * nalloc_base and nalloc_end variables. This space is currently 947 * being used for hat data structures required for tlb miss 948 * handling operations. We align nalloc_base to a l2 cache 949 * linesize because this is the line size the hardware uses to 950 * maintain cache coherency. 951 * 512K is carved out for module data. 952 */ 953 954 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 955 e_moddata = moddata + MODDATA; 956 nalloc_base = e_moddata; 957 958 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 959 valloc_base = nalloc_base; 960 961 /* 962 * Calculate the start of the data segment. 963 */ 964 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 965 prom_panic("nucleus data overflow"); 966 967 PRM_DEBUG(moddata); 968 PRM_DEBUG(nalloc_base); 969 PRM_DEBUG(nalloc_end); 970 971 /* 972 * Remember any slop after e_text so we can give it to the modules. 973 */ 974 PRM_DEBUG(e_text); 975 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 976 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 977 prom_panic("nucleus text overflow"); 978 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 979 modtext; 980 PRM_DEBUG(modtext); 981 PRM_DEBUG(modtext_sz); 982 983 init_boot_memlists(); 984 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 985 &boot_physavail, &boot_physavail_len, 986 &boot_virtavail, &boot_virtavail_len); 987 988 /* 989 * Remember what the physically available highest page is 990 * so that dumpsys works properly, and find out how much 991 * memory is installed. 992 */ 993 installed_top_size_memlist_array(boot_physinstalled, 994 boot_physinstalled_len, &physmax, &physinstalled); 995 PRM_DEBUG(physinstalled); 996 PRM_DEBUG(physmax); 997 998 /* Fill out memory nodes config structure */ 999 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1000 1001 /* 1002 * npages is the maximum of available physical memory possible. 1003 * (ie. it will never be more than this) 1004 * 1005 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1006 * using phys_avail will underestimate what will end up being freed. 1007 * A better initial guess is just total memory minus the kernel text 1008 */ 1009 npages = physinstalled - btop(MMU_PAGESIZE4M); 1010 1011 /* 1012 * First allocate things that can go in the nucleus data page 1013 * (fault status, TSBs, dmv, CPUs) 1014 */ 1015 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1016 1017 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1018 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1019 1020 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1021 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1022 1023 if (ndata_alloc_dmv(&ndata) != 0) 1024 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1025 1026 if (ndata_alloc_page_mutexs(&ndata) != 0) 1027 cmn_err(CE_PANIC, 1028 "no more nucleus memory after page free lists alloc"); 1029 1030 if (ndata_alloc_hat(&ndata, npages) != 0) 1031 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1032 1033 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1034 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1035 1036 /* 1037 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1038 * 1039 * There are comments all over the SFMMU code warning of dire 1040 * consequences if the TSBs are moved out of 32-bit space. This 1041 * is largely because the asm code uses "sethi %hi(addr)"-type 1042 * instructions which will not provide the expected result if the 1043 * address is a 64-bit one. 1044 * 1045 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1046 */ 1047 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1048 PRM_DEBUG(alloc_base); 1049 1050 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1051 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1052 PRM_DEBUG(alloc_base); 1053 1054 /* 1055 * Allocate IOMMU TSB array. We do this here so that the physical 1056 * memory gets deducted from the PROM's physical memory list. 1057 */ 1058 alloc_base = iommu_tsb_init(alloc_base); 1059 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1060 PRM_DEBUG(alloc_base); 1061 1062 /* 1063 * Allow for an early allocation of physically contiguous memory. 1064 */ 1065 alloc_base = contig_mem_prealloc(alloc_base, npages); 1066 1067 /* 1068 * Platforms like Starcat and OPL need special structures assigned in 1069 * 32-bit virtual address space because their probing routines execute 1070 * FCode, and FCode can't handle 64-bit virtual addresses... 1071 */ 1072 if (&plat_startup_memlist) { 1073 alloc_base = plat_startup_memlist(alloc_base); 1074 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1075 ecache_alignsize); 1076 PRM_DEBUG(alloc_base); 1077 } 1078 1079 /* 1080 * Save off where the contiguous allocations to date have ended 1081 * in econtig32. 1082 */ 1083 econtig32 = alloc_base; 1084 PRM_DEBUG(econtig32); 1085 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1086 cmn_err(CE_PANIC, "econtig32 too big"); 1087 1088 pp_sz = calc_pp_sz(npages); 1089 PRM_DEBUG(pp_sz); 1090 if (kpm_enable) { 1091 kpm_pp_sz = calc_kpmpp_sz(npages); 1092 PRM_DEBUG(kpm_pp_sz); 1093 } 1094 1095 hmehash_sz = calc_hmehash_sz(npages); 1096 PRM_DEBUG(hmehash_sz); 1097 1098 pagehash_sz = calc_pagehash_sz(npages); 1099 PRM_DEBUG(pagehash_sz); 1100 1101 pagelist_sz = calc_free_pagelist_sz(); 1102 PRM_DEBUG(pagelist_sz); 1103 1104 #ifdef TRAPTRACE 1105 tt_sz = calc_traptrace_sz(); 1106 PRM_DEBUG(tt_sz); 1107 #else 1108 tt_sz = 0; 1109 #endif /* TRAPTRACE */ 1110 1111 /* 1112 * Place the array that protects pp->p_selock in the kmem64 wad. 1113 */ 1114 pse_shift = size_pse_array(physmem, max_ncpus); 1115 PRM_DEBUG(pse_shift); 1116 pse_table_size = 1 << pse_shift; 1117 PRM_DEBUG(pse_table_size); 1118 psetable_sz = roundup( 1119 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1120 PRM_DEBUG(psetable_sz); 1121 1122 /* 1123 * Now allocate the whole wad 1124 */ 1125 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1126 pagelist_sz + tt_sz + psetable_sz; 1127 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1128 kmem64_base = (caddr_t)syslimit; 1129 kmem64_end = kmem64_base + kmem64_sz; 1130 alloc_kmem64(kmem64_base, kmem64_end); 1131 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1132 cmn_err(CE_PANIC, "not enough kmem64 space"); 1133 PRM_DEBUG(kmem64_base); 1134 PRM_DEBUG(kmem64_end); 1135 PRM_DEBUG(kmem64_aligned_end); 1136 1137 /* 1138 * ... and divy it up 1139 */ 1140 alloc_base = kmem64_base; 1141 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1142 pp_base = (page_t *)alloc_base; 1143 pp_sz = npages * sizeof (struct page); 1144 alloc_base += pp_sz; 1145 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1146 PRM_DEBUG(pp_base); 1147 PRM_DEBUG(npages); 1148 1149 if (kpm_enable) { 1150 kpm_pp_base = alloc_base; 1151 if (kpm_smallpages == 0) { 1152 /* kpm_npages based on physinstalled, don't reset */ 1153 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1154 } else { 1155 kpm_npages = ptokpmpr(npages); 1156 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1157 } 1158 alloc_base += kpm_pp_sz; 1159 alloc_base = 1160 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1161 PRM_DEBUG(kpm_pp_base); 1162 } 1163 1164 alloc_base = alloc_hmehash(alloc_base); 1165 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1166 PRM_DEBUG(alloc_base); 1167 1168 page_hash = (page_t **)alloc_base; 1169 alloc_base += pagehash_sz; 1170 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1171 PRM_DEBUG(page_hash); 1172 1173 alloc_base = alloc_page_freelists(alloc_base); 1174 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1175 PRM_DEBUG(alloc_base); 1176 1177 #ifdef TRAPTRACE 1178 ttrace_buf = alloc_base; 1179 alloc_base += tt_sz; 1180 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1181 PRM_DEBUG(alloc_base); 1182 #endif /* TRAPTRACE */ 1183 1184 pse_mutex = (pad_mutex_t *)alloc_base; 1185 alloc_base += psetable_sz; 1186 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1187 PRM_DEBUG(alloc_base); 1188 1189 /* adjust kmem64_end to what we really allocated */ 1190 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1191 kmem64_sz = kmem64_end - kmem64_base; 1192 1193 if (&ecache_init_scrub_flush_area) { 1194 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1195 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1196 } 1197 1198 /* 1199 * If physmem is patched to be non-zero, use it instead of 1200 * the monitor value unless physmem is larger than the total 1201 * amount of memory on hand. 1202 */ 1203 if (physmem == 0 || physmem > npages) 1204 physmem = npages; 1205 1206 /* 1207 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1208 * is mounted as root. This memory is held down by OBP and unlike 1209 * the stub boot_archive is never released. 1210 * 1211 * In order to get things sized correctly on lower memory 1212 * machines (where the memory used by the ramdisk represents 1213 * a significant portion of memory), physmem is adjusted. 1214 * 1215 * This is done by subtracting the ramdisk_size which is set 1216 * to the size of the ramdisk (in Kb) in /etc/system at the 1217 * time the miniroot archive is constructed. 1218 */ 1219 if (root_is_ramdisk == B_TRUE) 1220 physmem -= (ramdisk_size * 1024) / PAGESIZE; 1221 1222 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1223 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1224 1225 /* 1226 * Allocate space for the interrupt vector table. 1227 */ 1228 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1229 if (memspace != (caddr_t)intr_vec_table) 1230 prom_panic("interrupt vector table allocation failure"); 1231 1232 /* 1233 * Between now and when we finish copying in the memory lists, 1234 * allocations happen so the space gets fragmented and the 1235 * lists longer. Leave enough space for lists twice as 1236 * long as we have now; then roundup to a pagesize. 1237 */ 1238 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1239 prom_phys_avail_len() + prom_virt_avail_len()); 1240 memlist_sz *= 2; 1241 memlist_sz = roundup(memlist_sz, PAGESIZE); 1242 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1243 if (memspace == NULL) 1244 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1245 1246 memlist = (struct memlist *)memspace; 1247 memlist_end = (char *)memspace + memlist_sz; 1248 PRM_DEBUG(memlist); 1249 PRM_DEBUG(memlist_end); 1250 1251 PRM_DEBUG(sysbase); 1252 PRM_DEBUG(syslimit); 1253 kernelheap_init((void *)sysbase, (void *)syslimit, 1254 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1255 1256 /* 1257 * Take the most current snapshot we can by calling mem-update. 1258 */ 1259 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1260 &boot_physavail, &boot_physavail_len, 1261 &boot_virtavail, &boot_virtavail_len); 1262 1263 /* 1264 * Remove the space used by prom_alloc from the kernel heap 1265 * plus the area actually used by the OBP (if any) 1266 * ignoring virtual addresses in virt_avail, above syslimit. 1267 */ 1268 virt_avail = memlist; 1269 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1270 1271 for (cur = virt_avail; cur->next; cur = cur->next) { 1272 uint64_t range_base, range_size; 1273 1274 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1275 continue; 1276 if (range_base >= (uint64_t)syslimit) 1277 break; 1278 /* 1279 * Limit the range to end at syslimit. 1280 */ 1281 range_size = MIN(cur->next->address, 1282 (uint64_t)syslimit) - range_base; 1283 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1284 0, 0, (void *)range_base, (void *)(range_base + range_size), 1285 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1286 } 1287 1288 phys_avail = memlist; 1289 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1290 1291 /* 1292 * Add any extra memory at the end of the ndata region if there's at 1293 * least a page to add. There might be a few more pages available in 1294 * the middle of the ndata region, but for now they are ignored. 1295 */ 1296 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1297 if (nalloc_base == NULL) 1298 nalloc_base = nalloc_end; 1299 ndata_remain_sz = nalloc_end - nalloc_base; 1300 1301 /* 1302 * Copy physinstalled list into kernel space. 1303 */ 1304 phys_install = memlist; 1305 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1306 1307 /* 1308 * Create list of physical addrs we don't need pp's for: 1309 * kernel text 4M page 1310 * kernel data 4M page - ndata_remain_sz 1311 * kmem64 pages 1312 * 1313 * NB if adding any pages here, make sure no kpm page 1314 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1315 */ 1316 nopp_list = memlist; 1317 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1318 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1319 &memlist, &nopp_list); 1320 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1321 1322 if ((caddr_t)memlist > (memspace + memlist_sz)) 1323 prom_panic("memlist overflow"); 1324 1325 /* 1326 * Initialize the page structures from the memory lists. 1327 */ 1328 kphysm_init(); 1329 1330 availrmem_initial = availrmem = freemem; 1331 PRM_DEBUG(availrmem); 1332 1333 /* 1334 * Some of the locks depend on page_hashsz being set! 1335 * kmem_init() depends on this; so, keep it here. 1336 */ 1337 page_lock_init(); 1338 1339 /* 1340 * Initialize kernel memory allocator. 1341 */ 1342 kmem_init(); 1343 1344 /* 1345 * Factor in colorequiv to check additional 'equivalent' bins 1346 */ 1347 if (&page_set_colorequiv_arr_cpu != NULL) 1348 page_set_colorequiv_arr_cpu(); 1349 else 1350 page_set_colorequiv_arr(); 1351 1352 /* 1353 * Initialize bp_mapin(). 1354 */ 1355 bp_init(shm_alignment, HAT_STRICTORDER); 1356 1357 /* 1358 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1359 * vector data structures and MPO mblock structs from the 32-bit heap. 1360 */ 1361 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1362 panicbuf, panicbuf + PANICBUFSIZE, 1363 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1364 1365 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1366 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1367 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1368 1369 if (mpo_heap32_bufsz > (size_t)0) { 1370 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1371 PAGESIZE, 0, 0, mpo_heap32_buf, 1372 mpo_heap32_buf + mpo_heap32_bufsz, 1373 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1374 } 1375 mem_config_init(); 1376 } 1377 1378 static void 1379 startup_modules(void) 1380 { 1381 int nhblk1, nhblk8; 1382 size_t nhblksz; 1383 pgcnt_t pages_per_hblk; 1384 size_t hme8blk_sz, hme1blk_sz; 1385 1386 /* 1387 * Let the platforms have a chance to change default 1388 * values before reading system file. 1389 */ 1390 if (&set_platform_defaults) 1391 set_platform_defaults(); 1392 1393 /* 1394 * Calculate default settings of system parameters based upon 1395 * maxusers, yet allow to be overridden via the /etc/system file. 1396 */ 1397 param_calc(0); 1398 1399 mod_setup(); 1400 1401 /* 1402 * If this is a positron, complain and halt. 1403 */ 1404 if (&iam_positron && iam_positron()) { 1405 cmn_err(CE_WARN, "This hardware platform is not supported" 1406 " by this release of Solaris.\n"); 1407 #ifdef DEBUG 1408 prom_enter_mon(); /* Type 'go' to resume */ 1409 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1410 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1411 1412 #else /* DEBUG */ 1413 halt(0); 1414 #endif /* DEBUG */ 1415 } 1416 1417 /* 1418 * If we are running firmware that isn't 64-bit ready 1419 * then complain and halt. 1420 */ 1421 do_prom_version_check(); 1422 1423 /* 1424 * Initialize system parameters 1425 */ 1426 param_init(); 1427 1428 /* 1429 * maxmem is the amount of physical memory we're playing with. 1430 */ 1431 maxmem = physmem; 1432 1433 /* Set segkp limits. */ 1434 ncbase = kdi_segdebugbase; 1435 ncend = kdi_segdebugbase; 1436 1437 /* 1438 * Initialize the hat layer. 1439 */ 1440 hat_init(); 1441 1442 /* 1443 * Initialize segment management stuff. 1444 */ 1445 seg_init(); 1446 1447 /* 1448 * Create the va>tte handler, so the prom can understand 1449 * kernel translations. The handler is installed later, just 1450 * as we are about to take over the trap table from the prom. 1451 */ 1452 create_va_to_tte(); 1453 1454 /* 1455 * Load the forthdebugger (optional) 1456 */ 1457 forthdebug_init(); 1458 1459 /* 1460 * Create OBP node for console input callbacks 1461 * if it is needed. 1462 */ 1463 startup_create_io_node(); 1464 1465 if (modloadonly("fs", "specfs") == -1) 1466 halt("Can't load specfs"); 1467 1468 if (modloadonly("fs", "devfs") == -1) 1469 halt("Can't load devfs"); 1470 1471 if (modloadonly("misc", "swapgeneric") == -1) 1472 halt("Can't load swapgeneric"); 1473 1474 (void) modloadonly("sys", "lbl_edition"); 1475 1476 dispinit(); 1477 1478 /* 1479 * Infer meanings to the members of the idprom buffer. 1480 */ 1481 parse_idprom(); 1482 1483 /* Read cluster configuration data. */ 1484 clconf_init(); 1485 1486 setup_ddi(); 1487 1488 /* 1489 * Lets take this opportunity to load the root device. 1490 */ 1491 if (loadrootmodules() != 0) 1492 debug_enter("Can't load the root filesystem"); 1493 1494 /* 1495 * Load tod driver module for the tod part found on this system. 1496 * Recompute the cpu frequency/delays based on tod as tod part 1497 * tends to keep time more accurately. 1498 */ 1499 if (&load_tod_module) 1500 load_tod_module(); 1501 1502 /* 1503 * Allow platforms to load modules which might 1504 * be needed after bootops are gone. 1505 */ 1506 if (&load_platform_modules) 1507 load_platform_modules(); 1508 1509 setcpudelay(); 1510 1511 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1512 &boot_physavail, &boot_physavail_len, 1513 &boot_virtavail, &boot_virtavail_len); 1514 1515 /* 1516 * Calculation and allocation of hmeblks needed to remap 1517 * the memory allocated by PROM till now. 1518 * Overestimate the number of hblk1 elements by assuming 1519 * worst case of TTE64K mappings. 1520 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1521 */ 1522 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1523 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1524 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1525 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1526 1527 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1528 1529 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1530 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1531 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1532 1533 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1534 1535 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1536 nhblk8 = 0; 1537 while (bop_alloc_pages > 1) { 1538 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1539 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1540 bop_alloc_pages *= hme8blk_sz; 1541 bop_alloc_pages = btopr(bop_alloc_pages); 1542 } 1543 nhblk8 += 2; 1544 1545 /* 1546 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1547 * boundary, the number of hblk8's needed to map the entries in the 1548 * boot_virtavail list needs to be adjusted to take this into 1549 * consideration. Thus, we need to add additional hblk8's since it 1550 * is possible that an hblk8 will not have all 8 slots used due to 1551 * alignment constraints. Since there were boot_virtavail_len entries 1552 * in that list, we need to add that many hblk8's to the number 1553 * already calculated to make sure we don't underestimate. 1554 */ 1555 nhblk8 += boot_virtavail_len; 1556 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1557 1558 /* Allocate in pagesize chunks */ 1559 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1560 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1561 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1562 } 1563 1564 static void 1565 startup_bop_gone(void) 1566 { 1567 1568 /* 1569 * Destroy the MD initialized at startup 1570 * The startup initializes the MD framework 1571 * using prom and BOP alloc free it now. 1572 */ 1573 mach_descrip_startup_fini(); 1574 1575 /* 1576 * We're done with prom allocations. 1577 */ 1578 bop_fini(); 1579 1580 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1581 &boot_physavail, &boot_physavail_len, 1582 &boot_virtavail, &boot_virtavail_len); 1583 1584 /* 1585 * setup physically contiguous area twice as large as the ecache. 1586 * this is used while doing displacement flush of ecaches 1587 */ 1588 if (&ecache_flush_address) { 1589 ecache_flushaddr = ecache_flush_address(); 1590 if (ecache_flushaddr == (uint64_t)-1) { 1591 cmn_err(CE_PANIC, 1592 "startup: no memory to set ecache_flushaddr"); 1593 } 1594 } 1595 1596 /* 1597 * Virtual available next. 1598 */ 1599 ASSERT(virt_avail != NULL); 1600 memlist_free_list(virt_avail); 1601 virt_avail = memlist; 1602 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1603 1604 } 1605 1606 1607 /* 1608 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1609 * allocations have been performed. We can't call it in startup_bop_gone 1610 * since later operations can cause obp to allocate more memory. 1611 */ 1612 void 1613 startup_fixup_physavail(void) 1614 { 1615 struct memlist *cur; 1616 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1617 1618 PRM_DEBUG(kmem64_overmap_size); 1619 1620 /* 1621 * take the most current snapshot we can by calling mem-update 1622 */ 1623 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1624 &boot_physavail, &boot_physavail_len, 1625 &boot_virtavail, &boot_virtavail_len); 1626 1627 /* 1628 * Copy phys_avail list, again. 1629 * Both the kernel/boot and the prom have been allocating 1630 * from the original list we copied earlier. 1631 */ 1632 cur = memlist; 1633 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1634 1635 /* 1636 * Add any unused kmem64 memory from overmapped page 1637 * (Note: va_to_pa does not work for kmem64_end) 1638 */ 1639 if (kmem64_overmap_size) { 1640 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1641 kmem64_overmap_size, &memlist, &cur); 1642 } 1643 1644 /* 1645 * Add any extra memory after e_data we added to the phys_avail list 1646 * back to the old list. 1647 */ 1648 if (ndata_remain_sz >= MMU_PAGESIZE) 1649 memlist_add(va_to_pa(nalloc_base), 1650 (uint64_t)ndata_remain_sz, &memlist, &cur); 1651 1652 /* 1653 * There isn't any bounds checking on the memlist area 1654 * so ensure it hasn't overgrown. 1655 */ 1656 if ((caddr_t)memlist > (caddr_t)memlist_end) 1657 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1658 1659 /* 1660 * The kernel removes the pages that were allocated for it from 1661 * the freelist, but we now have to find any -extra- pages that 1662 * the prom has allocated for it's own book-keeping, and remove 1663 * them from the freelist too. sigh. 1664 */ 1665 sync_memlists(phys_avail, cur); 1666 1667 ASSERT(phys_avail != NULL); 1668 memlist_free_list(phys_avail); 1669 phys_avail = cur; 1670 1671 } 1672 1673 static void 1674 startup_vm(void) 1675 { 1676 size_t i; 1677 struct segmap_crargs a; 1678 struct segkpm_crargs b; 1679 1680 uint64_t avmem; 1681 caddr_t va; 1682 pgcnt_t max_phys_segkp; 1683 int mnode; 1684 1685 extern int use_brk_lpg, use_stk_lpg; 1686 1687 /* 1688 * get prom's mappings, create hments for them and switch 1689 * to the kernel context. 1690 */ 1691 hat_kern_setup(); 1692 1693 /* 1694 * Take over trap table 1695 */ 1696 setup_trap_table(); 1697 1698 /* 1699 * Install the va>tte handler, so that the prom can handle 1700 * misses and understand the kernel table layout in case 1701 * we need call into the prom. 1702 */ 1703 install_va_to_tte(); 1704 1705 /* 1706 * Set a flag to indicate that the tba has been taken over. 1707 */ 1708 tba_taken_over = 1; 1709 1710 /* initialize MMU primary context register */ 1711 mmu_init_kcontext(); 1712 1713 /* 1714 * The boot cpu can now take interrupts, x-calls, x-traps 1715 */ 1716 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1717 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1718 1719 /* 1720 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1721 */ 1722 tbr_wr_addr_inited = 1; 1723 1724 /* 1725 * Initialize VM system, and map kernel address space. 1726 */ 1727 kvm_init(); 1728 1729 /* 1730 * If the following is true, someone has patched 1731 * phsymem to be less than the number of pages that 1732 * the system actually has. Remove pages until system 1733 * memory is limited to the requested amount. Since we 1734 * have allocated page structures for all pages, we 1735 * correct the amount of memory we want to remove 1736 * by the size of the memory used to hold page structures 1737 * for the non-used pages. 1738 */ 1739 if (physmem < npages) { 1740 pgcnt_t diff, off; 1741 struct page *pp; 1742 struct seg kseg; 1743 1744 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1745 1746 off = 0; 1747 diff = npages - physmem; 1748 diff -= mmu_btopr(diff * sizeof (struct page)); 1749 kseg.s_as = &kas; 1750 while (diff--) { 1751 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1752 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1753 &kseg, (caddr_t)off); 1754 if (pp == NULL) 1755 cmn_err(CE_PANIC, "limited physmem too much!"); 1756 page_io_unlock(pp); 1757 page_downgrade(pp); 1758 availrmem--; 1759 off += MMU_PAGESIZE; 1760 } 1761 } 1762 1763 /* 1764 * When printing memory, show the total as physmem less 1765 * that stolen by a debugger. 1766 */ 1767 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1768 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1769 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1770 1771 avmem = (uint64_t)freemem << PAGESHIFT; 1772 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1773 1774 /* 1775 * For small memory systems disable automatic large pages. 1776 */ 1777 if (physmem < privm_lpg_min_physmem) { 1778 use_brk_lpg = 0; 1779 use_stk_lpg = 0; 1780 } 1781 1782 /* 1783 * Perform platform specific freelist processing 1784 */ 1785 if (&plat_freelist_process) { 1786 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1787 if (mem_node_config[mnode].exists) 1788 plat_freelist_process(mnode); 1789 } 1790 1791 /* 1792 * Initialize the segkp segment type. We position it 1793 * after the configured tables and buffers (whose end 1794 * is given by econtig) and before V_WKBASE_ADDR. 1795 * Also in this area is segkmap (size SEGMAPSIZE). 1796 */ 1797 1798 /* XXX - cache alignment? */ 1799 va = (caddr_t)SEGKPBASE; 1800 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1801 1802 max_phys_segkp = (physmem * 2); 1803 1804 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1805 segkpsize = btop(SEGKPDEFSIZE); 1806 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1807 "segkpsize has been reset to %ld pages", segkpsize); 1808 } 1809 1810 i = ptob(MIN(segkpsize, max_phys_segkp)); 1811 1812 rw_enter(&kas.a_lock, RW_WRITER); 1813 if (seg_attach(&kas, va, i, segkp) < 0) 1814 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1815 if (segkp_create(segkp) != 0) 1816 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1817 rw_exit(&kas.a_lock); 1818 1819 /* 1820 * kpm segment 1821 */ 1822 segmap_kpm = kpm_enable && 1823 segmap_kpm && PAGESIZE == MAXBSIZE; 1824 1825 if (kpm_enable) { 1826 rw_enter(&kas.a_lock, RW_WRITER); 1827 1828 /* 1829 * The segkpm virtual range range is larger than the 1830 * actual physical memory size and also covers gaps in 1831 * the physical address range for the following reasons: 1832 * . keep conversion between segkpm and physical addresses 1833 * simple, cheap and unambiguous. 1834 * . avoid extension/shrink of the the segkpm in case of DR. 1835 * . avoid complexity for handling of virtual addressed 1836 * caches, segkpm and the regular mapping scheme must be 1837 * kept in sync wrt. the virtual color of mapped pages. 1838 * Any accesses to virtual segkpm ranges not backed by 1839 * physical memory will fall through the memseg pfn hash 1840 * and will be handled in segkpm_fault. 1841 * Additional kpm_size spaces needed for vac alias prevention. 1842 */ 1843 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1844 segkpm) < 0) 1845 cmn_err(CE_PANIC, "cannot attach segkpm"); 1846 1847 b.prot = PROT_READ | PROT_WRITE; 1848 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1849 1850 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1851 panic("segkpm_create segkpm"); 1852 1853 rw_exit(&kas.a_lock); 1854 1855 mach_kpm_init(); 1856 } 1857 1858 if (!segzio_fromheap) { 1859 size_t size; 1860 size_t physmem_b = mmu_ptob(physmem); 1861 1862 /* size is in bytes, segziosize is in pages */ 1863 if (segziosize == 0) { 1864 size = physmem_b; 1865 } else { 1866 size = mmu_ptob(segziosize); 1867 } 1868 1869 if (size < SEGZIOMINSIZE) { 1870 size = SEGZIOMINSIZE; 1871 } else if (size > SEGZIOMAXSIZE) { 1872 size = SEGZIOMAXSIZE; 1873 /* 1874 * On 64-bit x86, we only have 2TB of KVA. This exists 1875 * for parity with x86. 1876 * 1877 * SEGZIOMAXSIZE is capped at 512gb so that segzio 1878 * doesn't consume all of KVA. However, if we have a 1879 * system that has more thant 512gb of physical memory, 1880 * we can actually consume about half of the difference 1881 * between 512gb and the rest of the available physical 1882 * memory. 1883 */ 1884 if (physmem_b > SEGZIOMAXSIZE) { 1885 size += (physmem_b - SEGZIOMAXSIZE) / 2; 1886 } 1887 } 1888 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 1889 /* put the base of the ZIO segment after the kpm segment */ 1890 segzio_base = kpm_vbase + (kpm_size * vac_colors); 1891 PRM_DEBUG(segziosize); 1892 PRM_DEBUG(segzio_base); 1893 1894 /* 1895 * On some platforms, kvm_init is called after the kpm 1896 * sizes have been determined. On SPARC, kvm_init is called 1897 * before, so we have to attach the kzioseg after kvm is 1898 * initialized, otherwise we'll try to allocate from the boot 1899 * area since the kernel heap hasn't yet been configured. 1900 */ 1901 rw_enter(&kas.a_lock, RW_WRITER); 1902 1903 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 1904 &kzioseg); 1905 (void) segkmem_zio_create(&kzioseg); 1906 1907 /* create zio area covering new segment */ 1908 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 1909 1910 rw_exit(&kas.a_lock); 1911 } 1912 1913 1914 /* 1915 * Now create generic mapping segment. This mapping 1916 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 1917 * virtual address is greater than the amount of free 1918 * memory that is available, then we trim back the 1919 * segment size to that amount 1920 */ 1921 va = (caddr_t)SEGMAPBASE; 1922 1923 /* 1924 * 1201049: segkmap base address must be MAXBSIZE aligned 1925 */ 1926 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 1927 1928 /* 1929 * Set size of segmap to percentage of freemem at boot, 1930 * but stay within the allowable range 1931 * Note we take percentage before converting from pages 1932 * to bytes to avoid an overflow on 32-bit kernels. 1933 */ 1934 i = mmu_ptob((freemem * segmap_percent) / 100); 1935 1936 if (i < MINMAPSIZE) 1937 i = MINMAPSIZE; 1938 1939 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 1940 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 1941 1942 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 1943 1944 rw_enter(&kas.a_lock, RW_WRITER); 1945 if (seg_attach(&kas, va, i, segkmap) < 0) 1946 cmn_err(CE_PANIC, "cannot attach segkmap"); 1947 1948 a.prot = PROT_READ | PROT_WRITE; 1949 a.shmsize = shm_alignment; 1950 a.nfreelist = 0; /* use segmap driver defaults */ 1951 1952 if (segmap_create(segkmap, (caddr_t)&a) != 0) 1953 panic("segmap_create segkmap"); 1954 rw_exit(&kas.a_lock); 1955 1956 segdev_init(); 1957 } 1958 1959 static void 1960 startup_end(void) 1961 { 1962 if ((caddr_t)memlist > (caddr_t)memlist_end) 1963 panic("memlist overflow 2"); 1964 memlist_free_block((caddr_t)memlist, 1965 ((caddr_t)memlist_end - (caddr_t)memlist)); 1966 memlist = NULL; 1967 1968 /* enable page_relocation since OBP is now done */ 1969 page_relocate_ready = 1; 1970 1971 /* 1972 * Perform tasks that get done after most of the VM 1973 * initialization has been done but before the clock 1974 * and other devices get started. 1975 */ 1976 kern_setup1(); 1977 1978 /* 1979 * Intialize the VM arenas for allocating physically 1980 * contiguus memory chunk for interrupt queues snd 1981 * allocate/register boot cpu's queues, if any and 1982 * allocate dump buffer for sun4v systems to store 1983 * extra crash information during crash dump 1984 */ 1985 contig_mem_init(); 1986 mach_descrip_init(); 1987 1988 if (cpu_intrq_setup(CPU)) { 1989 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 1990 } 1991 cpu_intrq_register(CPU); 1992 mach_htraptrace_setup(CPU->cpu_id); 1993 mach_htraptrace_configure(CPU->cpu_id); 1994 mach_dump_buffer_init(); 1995 1996 /* 1997 * Initialize interrupt related stuff 1998 */ 1999 cpu_intr_alloc(CPU, NINTR_THREADS); 2000 2001 (void) splzs(); /* allow hi clock ints but not zs */ 2002 2003 /* 2004 * Initialize errors. 2005 */ 2006 error_init(); 2007 2008 /* 2009 * Note that we may have already used kernel bcopy before this 2010 * point - but if you really care about this, adb the use_hw_* 2011 * variables to 0 before rebooting. 2012 */ 2013 mach_hw_copy_limit(); 2014 2015 /* 2016 * Install the "real" preemption guards before DDI services 2017 * are available. 2018 */ 2019 (void) prom_set_preprom(kern_preprom); 2020 (void) prom_set_postprom(kern_postprom); 2021 CPU->cpu_m.mutex_ready = 1; 2022 2023 /* 2024 * Initialize segnf (kernel support for non-faulting loads). 2025 */ 2026 segnf_init(); 2027 2028 /* 2029 * Configure the root devinfo node. 2030 */ 2031 configure(); /* set up devices */ 2032 mach_cpu_halt_idle(); 2033 } 2034 2035 2036 void 2037 post_startup(void) 2038 { 2039 #ifdef PTL1_PANIC_DEBUG 2040 extern void init_ptl1_thread(void); 2041 #endif /* PTL1_PANIC_DEBUG */ 2042 extern void abort_sequence_init(void); 2043 2044 /* 2045 * Set the system wide, processor-specific flags to be passed 2046 * to userland via the aux vector for performance hints and 2047 * instruction set extensions. 2048 */ 2049 bind_hwcap(); 2050 2051 /* 2052 * Startup memory scrubber (if any) 2053 */ 2054 mach_memscrub(); 2055 2056 /* 2057 * Allocate soft interrupt to handle abort sequence. 2058 */ 2059 abort_sequence_init(); 2060 2061 /* 2062 * Configure the rest of the system. 2063 * Perform forceloading tasks for /etc/system. 2064 */ 2065 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2066 /* 2067 * ON4.0: Force /proc module in until clock interrupt handle fixed 2068 * ON4.0: This must be fixed or restated in /etc/systems. 2069 */ 2070 (void) modload("fs", "procfs"); 2071 2072 /* load machine class specific drivers */ 2073 load_mach_drivers(); 2074 2075 /* load platform specific drivers */ 2076 if (&load_platform_drivers) 2077 load_platform_drivers(); 2078 2079 /* load vis simulation module, if we are running w/fpu off */ 2080 if (!fpu_exists) { 2081 if (modload("misc", "vis") == -1) 2082 halt("Can't load vis"); 2083 } 2084 2085 mach_fpras(); 2086 2087 maxmem = freemem; 2088 2089 #ifdef PTL1_PANIC_DEBUG 2090 init_ptl1_thread(); 2091 #endif /* PTL1_PANIC_DEBUG */ 2092 2093 if (&cif_init) 2094 cif_init(); 2095 } 2096 2097 #ifdef PTL1_PANIC_DEBUG 2098 int ptl1_panic_test = 0; 2099 int ptl1_panic_xc_one_test = 0; 2100 int ptl1_panic_xc_all_test = 0; 2101 int ptl1_panic_xt_one_test = 0; 2102 int ptl1_panic_xt_all_test = 0; 2103 kthread_id_t ptl1_thread_p = NULL; 2104 kcondvar_t ptl1_cv; 2105 kmutex_t ptl1_mutex; 2106 int ptl1_recurse_count_threshold = 0x40; 2107 int ptl1_recurse_trap_threshold = 0x3d; 2108 extern void ptl1_recurse(int, int); 2109 extern void ptl1_panic_xt(int, int); 2110 2111 /* 2112 * Called once per second by timeout() to wake up 2113 * the ptl1_panic thread to see if it should cause 2114 * a trap to the ptl1_panic() code. 2115 */ 2116 /* ARGSUSED */ 2117 static void 2118 ptl1_wakeup(void *arg) 2119 { 2120 mutex_enter(&ptl1_mutex); 2121 cv_signal(&ptl1_cv); 2122 mutex_exit(&ptl1_mutex); 2123 } 2124 2125 /* 2126 * ptl1_panic cross call function: 2127 * Needed because xc_one() and xc_some() can pass 2128 * 64 bit args but ptl1_recurse() expects ints. 2129 */ 2130 static void 2131 ptl1_panic_xc(void) 2132 { 2133 ptl1_recurse(ptl1_recurse_count_threshold, 2134 ptl1_recurse_trap_threshold); 2135 } 2136 2137 /* 2138 * The ptl1 thread waits for a global flag to be set 2139 * and uses the recurse thresholds to set the stack depth 2140 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2141 * or indirectly via the cross call and cross trap functions. 2142 * 2143 * This is useful testing stack overflows and normal 2144 * ptl1_panic() states with a know stack frame. 2145 * 2146 * ptl1_recurse() is an asm function in ptl1_panic.s that 2147 * sets the {In, Local, Out, and Global} registers to a 2148 * know state on the stack and just prior to causing a 2149 * test ptl1_panic trap. 2150 */ 2151 static void 2152 ptl1_thread(void) 2153 { 2154 mutex_enter(&ptl1_mutex); 2155 while (ptl1_thread_p) { 2156 cpuset_t other_cpus; 2157 int cpu_id; 2158 int my_cpu_id; 2159 int target_cpu_id; 2160 int target_found; 2161 2162 if (ptl1_panic_test) { 2163 ptl1_recurse(ptl1_recurse_count_threshold, 2164 ptl1_recurse_trap_threshold); 2165 } 2166 2167 /* 2168 * Find potential targets for x-call and x-trap, 2169 * if any exist while preempt is disabled we 2170 * start a ptl1_panic if requested via a 2171 * globals. 2172 */ 2173 kpreempt_disable(); 2174 my_cpu_id = CPU->cpu_id; 2175 other_cpus = cpu_ready_set; 2176 CPUSET_DEL(other_cpus, CPU->cpu_id); 2177 target_found = 0; 2178 if (!CPUSET_ISNULL(other_cpus)) { 2179 /* 2180 * Pick the first one 2181 */ 2182 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2183 if (cpu_id == my_cpu_id) 2184 continue; 2185 2186 if (CPU_XCALL_READY(cpu_id)) { 2187 target_cpu_id = cpu_id; 2188 target_found = 1; 2189 break; 2190 } 2191 } 2192 ASSERT(target_found); 2193 2194 if (ptl1_panic_xc_one_test) { 2195 xc_one(target_cpu_id, 2196 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2197 } 2198 if (ptl1_panic_xc_all_test) { 2199 xc_some(other_cpus, 2200 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2201 } 2202 if (ptl1_panic_xt_one_test) { 2203 xt_one(target_cpu_id, 2204 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2205 } 2206 if (ptl1_panic_xt_all_test) { 2207 xt_some(other_cpus, 2208 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2209 } 2210 } 2211 kpreempt_enable(); 2212 (void) timeout(ptl1_wakeup, NULL, hz); 2213 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2214 } 2215 mutex_exit(&ptl1_mutex); 2216 } 2217 2218 /* 2219 * Called during early startup to create the ptl1_thread 2220 */ 2221 void 2222 init_ptl1_thread(void) 2223 { 2224 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2225 &p0, TS_RUN, 0); 2226 } 2227 #endif /* PTL1_PANIC_DEBUG */ 2228 2229 2230 static void 2231 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2232 { 2233 struct memlist *new; 2234 2235 new = *memlistp; 2236 new->address = start; 2237 new->size = len; 2238 *memlistp = new + 1; 2239 } 2240 2241 /* 2242 * Add to a memory list. 2243 * start = start of new memory segment 2244 * len = length of new memory segment in bytes 2245 * memlistp = pointer to array of available memory segment structures 2246 * curmemlistp = memory list to which to add segment. 2247 */ 2248 static void 2249 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2250 struct memlist **curmemlistp) 2251 { 2252 struct memlist *new = *memlistp; 2253 2254 memlist_new(start, len, memlistp); 2255 memlist_insert(new, curmemlistp); 2256 } 2257 2258 static int 2259 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2260 { 2261 int nseg; 2262 size_t memseg_sz; 2263 struct memseg *msp; 2264 2265 /* 2266 * The memseg list is for the chunks of physical memory that 2267 * will be managed by the vm system. The number calculated is 2268 * a guess as boot may fragment it more when memory allocations 2269 * are made before kphysm_init(). 2270 */ 2271 memseg_sz = (avail + 10) * sizeof (struct memseg); 2272 memseg_sz = roundup(memseg_sz, PAGESIZE); 2273 nseg = memseg_sz / sizeof (struct memseg); 2274 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2275 if (msp == NULL) 2276 return (1); 2277 PRM_DEBUG(memseg_free); 2278 2279 while (nseg--) { 2280 msp->next = memseg_free; 2281 memseg_free = msp; 2282 msp++; 2283 } 2284 return (0); 2285 } 2286 2287 /* 2288 * In the case of architectures that support dynamic addition of 2289 * memory at run-time there are two cases where memsegs need to 2290 * be initialized and added to the memseg list. 2291 * 1) memsegs that are constructed at startup. 2292 * 2) memsegs that are constructed at run-time on 2293 * hot-plug capable architectures. 2294 * This code was originally part of the function kphysm_init(). 2295 */ 2296 2297 static void 2298 memseg_list_add(struct memseg *memsegp) 2299 { 2300 struct memseg **prev_memsegp; 2301 pgcnt_t num; 2302 2303 /* insert in memseg list, decreasing number of pages order */ 2304 2305 num = MSEG_NPAGES(memsegp); 2306 2307 for (prev_memsegp = &memsegs; *prev_memsegp; 2308 prev_memsegp = &((*prev_memsegp)->next)) { 2309 if (num > MSEG_NPAGES(*prev_memsegp)) 2310 break; 2311 } 2312 2313 memsegp->next = *prev_memsegp; 2314 *prev_memsegp = memsegp; 2315 2316 if (kpm_enable) { 2317 memsegp->nextpa = (memsegp->next) ? 2318 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2319 2320 if (prev_memsegp != &memsegs) { 2321 struct memseg *msp; 2322 msp = (struct memseg *)((caddr_t)prev_memsegp - 2323 offsetof(struct memseg, next)); 2324 msp->nextpa = va_to_pa(memsegp); 2325 } else { 2326 memsegspa = va_to_pa(memsegs); 2327 } 2328 } 2329 } 2330 2331 /* 2332 * PSM add_physmem_cb(). US-II and newer processors have some 2333 * flavor of the prefetch capability implemented. We exploit 2334 * this capability for optimum performance. 2335 */ 2336 #define PREFETCH_BYTES 64 2337 2338 void 2339 add_physmem_cb(page_t *pp, pfn_t pnum) 2340 { 2341 extern void prefetch_page_w(void *); 2342 2343 pp->p_pagenum = pnum; 2344 2345 /* 2346 * Prefetch one more page_t into E$. To prevent future 2347 * mishaps with the sizeof(page_t) changing on us, we 2348 * catch this on debug kernels if we can't bring in the 2349 * entire hpage with 2 PREFETCH_BYTES reads. See 2350 * also, sun4u/cpu/cpu_module.c 2351 */ 2352 /*LINTED*/ 2353 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2354 prefetch_page_w((char *)pp); 2355 } 2356 2357 /* 2358 * Find memseg with given pfn 2359 */ 2360 static struct memseg * 2361 memseg_find(pfn_t base, pfn_t *next) 2362 { 2363 struct memseg *seg; 2364 2365 if (next != NULL) 2366 *next = LONG_MAX; 2367 for (seg = memsegs; seg != NULL; seg = seg->next) { 2368 if (base >= seg->pages_base && base < seg->pages_end) 2369 return (seg); 2370 if (next != NULL && seg->pages_base > base && 2371 seg->pages_base < *next) 2372 *next = seg->pages_base; 2373 } 2374 return (NULL); 2375 } 2376 2377 extern struct vnode prom_ppages; 2378 2379 /* 2380 * Put page allocated by OBP on prom_ppages 2381 */ 2382 static void 2383 kphysm_erase(uint64_t addr, uint64_t len) 2384 { 2385 struct page *pp; 2386 struct memseg *seg; 2387 pfn_t base = btop(addr), next; 2388 pgcnt_t num = btop(len); 2389 2390 while (num != 0) { 2391 pgcnt_t off, left; 2392 2393 seg = memseg_find(base, &next); 2394 if (seg == NULL) { 2395 if (next == LONG_MAX) 2396 break; 2397 left = MIN(next - base, num); 2398 base += left, num -= left; 2399 continue; 2400 } 2401 off = base - seg->pages_base; 2402 pp = seg->pages + off; 2403 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2404 while (num != left) { 2405 /* 2406 * init it, lock it, and hashin on prom_pages vp. 2407 * 2408 * XXX vnode offsets on the prom_ppages vnode 2409 * are page numbers (gack) for >32 bit 2410 * physical memory machines. 2411 */ 2412 add_physmem_cb(pp, base); 2413 if (page_trylock(pp, SE_EXCL) == 0) 2414 cmn_err(CE_PANIC, "prom page locked"); 2415 (void) page_hashin(pp, &prom_ppages, 2416 (offset_t)base, NULL); 2417 (void) page_pp_lock(pp, 0, 1); 2418 pp++, base++, num--; 2419 } 2420 } 2421 } 2422 2423 static page_t *ppnext; 2424 static pgcnt_t ppleft; 2425 2426 static void *kpm_ppnext; 2427 static pgcnt_t kpm_ppleft; 2428 2429 /* 2430 * Create a memseg 2431 */ 2432 static void 2433 kphysm_memseg(uint64_t addr, uint64_t len) 2434 { 2435 pfn_t base = btop(addr); 2436 pgcnt_t num = btop(len); 2437 struct memseg *seg; 2438 2439 seg = memseg_free; 2440 memseg_free = seg->next; 2441 ASSERT(seg != NULL); 2442 2443 seg->pages = ppnext; 2444 seg->epages = ppnext + num; 2445 seg->pages_base = base; 2446 seg->pages_end = base + num; 2447 ppnext += num; 2448 ppleft -= num; 2449 2450 if (kpm_enable) { 2451 pgcnt_t kpnum = ptokpmpr(num); 2452 2453 if (kpnum > kpm_ppleft) 2454 panic("kphysm_memseg: kpm_pp overflow"); 2455 seg->pagespa = va_to_pa(seg->pages); 2456 seg->epagespa = va_to_pa(seg->epages); 2457 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2458 seg->kpm_nkpmpgs = kpnum; 2459 /* 2460 * In the kpm_smallpage case, the kpm array 2461 * is 1-1 wrt the page array 2462 */ 2463 if (kpm_smallpages) { 2464 kpm_spage_t *kpm_pp = kpm_ppnext; 2465 2466 kpm_ppnext = kpm_pp + kpnum; 2467 seg->kpm_spages = kpm_pp; 2468 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2469 } else { 2470 kpm_page_t *kpm_pp = kpm_ppnext; 2471 2472 kpm_ppnext = kpm_pp + kpnum; 2473 seg->kpm_pages = kpm_pp; 2474 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2475 /* ASSERT no kpm overlaps */ 2476 ASSERT( 2477 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2478 ASSERT(memseg_find( 2479 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2480 } 2481 kpm_ppleft -= num; 2482 } 2483 2484 memseg_list_add(seg); 2485 } 2486 2487 /* 2488 * Add range to free list 2489 */ 2490 void 2491 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2492 { 2493 struct page *pp; 2494 struct memseg *seg; 2495 pfn_t base = btop(addr); 2496 pgcnt_t num = btop(len); 2497 2498 seg = memseg_find(base, NULL); 2499 ASSERT(seg != NULL); 2500 pp = seg->pages + (base - seg->pages_base); 2501 2502 if (reclaim) { 2503 struct page *rpp = pp; 2504 struct page *lpp = pp + num; 2505 2506 /* 2507 * page should be locked on prom_ppages 2508 * unhash and unlock it 2509 */ 2510 while (rpp < lpp) { 2511 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages); 2512 page_pp_unlock(rpp, 0, 1); 2513 page_hashout(rpp, NULL); 2514 page_unlock(rpp); 2515 rpp++; 2516 } 2517 } 2518 2519 /* 2520 * add_physmem() initializes the PSM part of the page 2521 * struct by calling the PSM back with add_physmem_cb(). 2522 * In addition it coalesces pages into larger pages as 2523 * it initializes them. 2524 */ 2525 add_physmem(pp, num, base); 2526 } 2527 2528 /* 2529 * kphysm_init() tackles the problem of initializing physical memory. 2530 */ 2531 static void 2532 kphysm_init(void) 2533 { 2534 struct memlist *pmem; 2535 2536 ASSERT(page_hash != NULL && page_hashsz != 0); 2537 2538 ppnext = pp_base; 2539 ppleft = npages; 2540 kpm_ppnext = kpm_pp_base; 2541 kpm_ppleft = kpm_npages; 2542 2543 /* 2544 * installed pages not on nopp_memlist go in memseg list 2545 */ 2546 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2547 2548 /* 2549 * Free the avail list 2550 */ 2551 for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 2552 kphysm_add(pmem->address, pmem->size, 0); 2553 2554 /* 2555 * Erase pages that aren't available 2556 */ 2557 diff_memlists(phys_install, phys_avail, kphysm_erase); 2558 2559 build_pfn_hash(); 2560 } 2561 2562 /* 2563 * Kernel VM initialization. 2564 * Assumptions about kernel address space ordering: 2565 * (1) gap (user space) 2566 * (2) kernel text 2567 * (3) kernel data/bss 2568 * (4) gap 2569 * (5) kernel data structures 2570 * (6) gap 2571 * (7) debugger (optional) 2572 * (8) monitor 2573 * (9) gap (possibly null) 2574 * (10) dvma 2575 * (11) devices 2576 */ 2577 static void 2578 kvm_init(void) 2579 { 2580 /* 2581 * Put the kernel segments in kernel address space. 2582 */ 2583 rw_enter(&kas.a_lock, RW_WRITER); 2584 as_avlinit(&kas); 2585 2586 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2587 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2588 (void) segkmem_create(&ktextseg); 2589 2590 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2591 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2592 (void) segkmem_create(&ktexthole); 2593 2594 (void) seg_attach(&kas, (caddr_t)valloc_base, 2595 (size_t)(econtig32 - valloc_base), &kvalloc); 2596 (void) segkmem_create(&kvalloc); 2597 2598 if (kmem64_base) { 2599 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2600 (size_t)(kmem64_end - kmem64_base), &kmem64); 2601 (void) segkmem_create(&kmem64); 2602 } 2603 2604 /* 2605 * We're about to map out /boot. This is the beginning of the 2606 * system resource management transition. We can no longer 2607 * call into /boot for I/O or memory allocations. 2608 */ 2609 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2610 (void) segkmem_create(&kvseg); 2611 hblk_alloc_dynamic = 1; 2612 2613 /* 2614 * we need to preallocate pages for DR operations before enabling large 2615 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2616 */ 2617 memseg_remap_init(); 2618 2619 /* at this point we are ready to use large page heap */ 2620 segkmem_heap_lp_init(); 2621 2622 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2623 &kvseg32); 2624 (void) segkmem_create(&kvseg32); 2625 2626 /* 2627 * Create a segment for the debugger. 2628 */ 2629 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2630 (void) segkmem_create(&kdebugseg); 2631 2632 rw_exit(&kas.a_lock); 2633 } 2634 2635 char obp_tte_str[] = 2636 "h# %x constant MMU_PAGESHIFT " 2637 "h# %x constant TTE8K " 2638 "h# %x constant SFHME_SIZE " 2639 "h# %x constant SFHME_TTE " 2640 "h# %x constant HMEBLK_TAG " 2641 "h# %x constant HMEBLK_NEXT " 2642 "h# %x constant HMEBLK_MISC " 2643 "h# %x constant HMEBLK_HME1 " 2644 "h# %x constant NHMENTS " 2645 "h# %x constant HBLK_SZMASK " 2646 "h# %x constant HBLK_RANGE_SHIFT " 2647 "h# %x constant HMEBP_HBLK " 2648 "h# %x constant HMEBUCKET_SIZE " 2649 "h# %x constant HTAG_SFMMUPSZ " 2650 "h# %x constant HTAG_BSPAGE_SHIFT " 2651 "h# %x constant HTAG_REHASH_SHIFT " 2652 "h# %x constant SFMMU_INVALID_SHMERID " 2653 "h# %x constant mmu_hashcnt " 2654 "h# %p constant uhme_hash " 2655 "h# %p constant khme_hash " 2656 "h# %x constant UHMEHASH_SZ " 2657 "h# %x constant KHMEHASH_SZ " 2658 "h# %p constant KCONTEXT " 2659 "h# %p constant KHATID " 2660 "h# %x constant ASI_MEM " 2661 2662 ": PHYS-X@ ( phys -- data ) " 2663 " ASI_MEM spacex@ " 2664 "; " 2665 2666 ": PHYS-W@ ( phys -- data ) " 2667 " ASI_MEM spacew@ " 2668 "; " 2669 2670 ": PHYS-L@ ( phys -- data ) " 2671 " ASI_MEM spaceL@ " 2672 "; " 2673 2674 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2675 " 3 * MMU_PAGESHIFT + " 2676 "; " 2677 2678 ": TTE_IS_VALID ( ttep -- flag ) " 2679 " PHYS-X@ 0< " 2680 "; " 2681 2682 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2683 " dup TTE8K = if " 2684 " drop HBLK_RANGE_SHIFT " 2685 " else " 2686 " TTE_PAGE_SHIFT " 2687 " then " 2688 "; " 2689 2690 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2691 " tuck >> swap MMU_PAGESHIFT - << " 2692 "; " 2693 2694 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2695 " >> over xor swap ( hash sfmmup ) " 2696 " KHATID <> if ( hash ) " 2697 " UHMEHASH_SZ and ( bucket ) " 2698 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2699 " else ( hash ) " 2700 " KHMEHASH_SZ and ( bucket ) " 2701 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2702 " then ( hmebp ) " 2703 "; " 2704 2705 ": HME_HASH_TABLE_SEARCH " 2706 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2707 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2708 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2709 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2710 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2711 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2712 " else " 2713 " hmeblk_next + phys-x@ false " 2714 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2715 " then " 2716 " else " 2717 " hmeblk_next + phys-x@ false " 2718 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2719 " then " 2720 " else " 2721 " true " 2722 " then " 2723 " until r> drop " 2724 "; " 2725 2726 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2727 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2728 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2729 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2730 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2731 "; " 2732 2733 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2734 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2735 " TTE8K = if ( hmeblkp addr ) " 2736 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2737 " else ( hmeblkp addr ) " 2738 " drop 0 ( hmeblkp 0 ) " 2739 " then ( hmeblkp hme-index ) " 2740 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2741 " SFHME_TTE + ( ttep ) " 2742 "; " 2743 2744 ": unix-tte ( addr cnum -- false | tte-data true ) " 2745 " KCONTEXT = if ( addr ) " 2746 " KHATID ( addr khatid ) " 2747 " else ( addr ) " 2748 " drop false exit ( false ) " 2749 " then " 2750 " ( addr khatid ) " 2751 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2752 " 2dup swap i HME_HASH_SHIFT " 2753 "( addr sfmmup sfmmup addr hmeshift ) " 2754 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2755 " over i 4 pick " 2756 "( addr sfmmup hmebp sfmmup rehash addr ) " 2757 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2758 " HME_HASH_TABLE_SEARCH " 2759 "( addr sfmmup { null | hmeblkp } ) " 2760 " ?dup if ( addr sfmmup hmeblkp ) " 2761 " nip swap HBLK_TO_TTEP ( ttep ) " 2762 " dup TTE_IS_VALID if ( valid-ttep ) " 2763 " PHYS-X@ true ( tte-data true ) " 2764 " else ( invalid-tte ) " 2765 " drop false ( false ) " 2766 " then ( false | tte-data true ) " 2767 " unloop exit ( false | tte-data true ) " 2768 " then ( addr sfmmup ) " 2769 " loop ( addr sfmmup ) " 2770 " 2drop false ( false ) " 2771 "; " 2772 ; 2773 2774 void 2775 create_va_to_tte(void) 2776 { 2777 char *bp; 2778 extern int khmehash_num, uhmehash_num; 2779 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2780 2781 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2782 2783 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2784 2785 /* 2786 * Teach obp how to parse our sw ttes. 2787 */ 2788 (void) sprintf(bp, obp_tte_str, 2789 MMU_PAGESHIFT, 2790 TTE8K, 2791 sizeof (struct sf_hment), 2792 OFFSET(struct sf_hment, hme_tte), 2793 OFFSET(struct hme_blk, hblk_tag), 2794 OFFSET(struct hme_blk, hblk_nextpa), 2795 OFFSET(struct hme_blk, hblk_misc), 2796 OFFSET(struct hme_blk, hblk_hme), 2797 NHMENTS, 2798 HBLK_SZMASK, 2799 HBLK_RANGE_SHIFT, 2800 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2801 sizeof (struct hmehash_bucket), 2802 HTAG_SFMMUPSZ, 2803 HTAG_BSPAGE_SHIFT, 2804 HTAG_REHASH_SHIFT, 2805 SFMMU_INVALID_SHMERID, 2806 mmu_hashcnt, 2807 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2808 (caddr_t)va_to_pa((caddr_t)khme_hash), 2809 UHMEHASH_SZ, 2810 KHMEHASH_SZ, 2811 KCONTEXT, 2812 KHATID, 2813 ASI_MEM); 2814 prom_interpret(bp, 0, 0, 0, 0, 0); 2815 2816 kobj_free(bp, MMU_PAGESIZE); 2817 } 2818 2819 void 2820 install_va_to_tte(void) 2821 { 2822 /* 2823 * advise prom that he can use unix-tte 2824 */ 2825 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2826 } 2827 2828 /* 2829 * Here we add "device-type=console" for /os-io node, for currently 2830 * our kernel console output only supports displaying text and 2831 * performing cursor-positioning operations (through kernel framebuffer 2832 * driver) and it doesn't support other functionalities required for a 2833 * standard "display" device as specified in 1275 spec. The main missing 2834 * interface defined by the 1275 spec is "draw-logo". 2835 * also see the comments above prom_stdout_is_framebuffer(). 2836 */ 2837 static char *create_node = 2838 "\" /\" find-device " 2839 "new-device " 2840 "\" os-io\" device-name " 2841 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 2842 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2843 " 2>r swap 2 2r> ['] $callback catch if " 2844 " 2drop 3drop 0 " 2845 " then " 2846 "; " 2847 ": read ( adr,len -- #read ) " 2848 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2849 " ( retN ... ret1 N ) " 2850 " ?dup if " 2851 " swap >r 1- 0 ?do drop loop r> " 2852 " else " 2853 " -2 " 2854 " then " 2855 "; " 2856 ": write ( adr,len -- #written ) " 2857 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2858 " ( retN ... ret1 N ) " 2859 " ?dup if " 2860 " swap >r 1- 0 ?do drop loop r> " 2861 " else " 2862 " 0 " 2863 " then " 2864 "; " 2865 ": poll-tty ( -- ) ; " 2866 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2867 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2868 ": cb-give/take ( $method -- ) " 2869 " 0 -rot ['] $callback catch ?dup if " 2870 " >r 2drop 2drop r> throw " 2871 " else " 2872 " 0 ?do drop loop " 2873 " then " 2874 "; " 2875 ": give ( -- ) \" exit-input\" cb-give/take ; " 2876 ": take ( -- ) \" enter-input\" cb-give/take ; " 2877 ": open ( -- ok? ) true ; " 2878 ": close ( -- ) ; " 2879 "finish-device " 2880 "device-end "; 2881 2882 /* 2883 * Create the OBP input/output node (FCode serial driver). 2884 * It is needed for both USB console keyboard and for 2885 * the kernel terminal emulator. It is too early to check for a 2886 * kernel console compatible framebuffer now, so we create this 2887 * so that we're ready if we need to enable kernel terminal emulation. 2888 * 2889 * When the USB software takes over the input device at the time 2890 * consconfig runs, OBP's stdin is redirected to this node. 2891 * Whenever the FORTH user interface is used after this switch, 2892 * the node will call back into the kernel for console input. 2893 * If a serial device such as ttya or a UART with a Type 5 keyboard 2894 * attached is used, OBP takes over the serial device when the system 2895 * goes to the debugger after the system is booted. This sharing 2896 * of the relatively simple serial device is difficult but possible. 2897 * Sharing the USB host controller is impossible due its complexity. 2898 * 2899 * Similarly to USB keyboard input redirection, after consconfig_dacf 2900 * configures a kernel console framebuffer as the standard output 2901 * device, OBP's stdout is switched to to vector through the 2902 * /os-io node into the kernel terminal emulator. 2903 */ 2904 static void 2905 startup_create_io_node(void) 2906 { 2907 prom_interpret(create_node, 0, 0, 0, 0, 0); 2908 } 2909 2910 2911 static void 2912 do_prom_version_check(void) 2913 { 2914 int i; 2915 pnode_t node; 2916 char buf[64]; 2917 static char drev[] = "Down-rev firmware detected%s\n" 2918 "\tPlease upgrade to the following minimum version:\n" 2919 "\t\t%s\n"; 2920 2921 i = prom_version_check(buf, sizeof (buf), &node); 2922 2923 if (i == PROM_VER64_OK) 2924 return; 2925 2926 if (i == PROM_VER64_UPGRADE) { 2927 cmn_err(CE_WARN, drev, "", buf); 2928 2929 #ifdef DEBUG 2930 prom_enter_mon(); /* Type 'go' to continue */ 2931 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 2932 return; 2933 #else 2934 halt(0); 2935 #endif 2936 } 2937 2938 /* 2939 * The other possibility is that this is a server running 2940 * good firmware, but down-rev firmware was detected on at 2941 * least one other cpu board. We just complain if we see 2942 * that. 2943 */ 2944 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 2945 } 2946 2947 2948 /* 2949 * Must be defined in platform dependent code. 2950 */ 2951 extern caddr_t modtext; 2952 extern size_t modtext_sz; 2953 extern caddr_t moddata; 2954 2955 #define HEAPTEXT_ARENA(addr) \ 2956 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 2957 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 2958 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 2959 2960 #define HEAPTEXT_OVERSIZED(addr) \ 2961 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 2962 2963 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 2964 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 2965 kmutex_t texthole_lock; 2966 2967 char kern_bootargs[OBP_MAXPATHLEN]; 2968 2969 void 2970 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 2971 { 2972 uintptr_t addr, limit; 2973 2974 addr = HEAPTEXT_BASE; 2975 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 2976 2977 /* 2978 * Before we initialize the text_arena, we want to punch holes in the 2979 * underlying heaptext_arena. This guarantees that for any text 2980 * address we can find a text hole less than HEAPTEXT_MAPPED away. 2981 */ 2982 for (; addr + HEAPTEXT_UNMAPPED <= limit; 2983 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 2984 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 2985 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 2986 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2987 } 2988 2989 /* 2990 * Allocate one page at the oversize to break up the text region 2991 * from the oversized region. 2992 */ 2993 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 2994 (void *)limit, (void *)(limit + PAGESIZE), 2995 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 2996 2997 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 2998 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 2999 heaptext_arena, 0, VM_SLEEP); 3000 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3001 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3002 } 3003 3004 caddr_t 3005 kobj_text_alloc(vmem_t *arena, size_t size) 3006 { 3007 caddr_t rval, better; 3008 3009 /* 3010 * First, try a sleeping allocation. 3011 */ 3012 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3013 3014 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3015 return (rval); 3016 3017 /* 3018 * We didn't get the area that we wanted. We're going to try to do an 3019 * allocation with explicit constraints. 3020 */ 3021 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3022 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3023 VM_NOSLEEP | VM_BESTFIT); 3024 3025 if (better != NULL) { 3026 /* 3027 * That worked. Free our first attempt and return. 3028 */ 3029 vmem_free(arena, rval, size); 3030 return (better); 3031 } 3032 3033 /* 3034 * That didn't work; we'll have to return our first attempt. 3035 */ 3036 return (rval); 3037 } 3038 3039 caddr_t 3040 kobj_texthole_alloc(caddr_t addr, size_t size) 3041 { 3042 int arena = HEAPTEXT_ARENA(addr); 3043 char c[30]; 3044 uintptr_t base; 3045 3046 if (HEAPTEXT_OVERSIZED(addr)) { 3047 /* 3048 * If this is an oversized allocation, there is no text hole 3049 * available for it; return NULL. 3050 */ 3051 return (NULL); 3052 } 3053 3054 mutex_enter(&texthole_lock); 3055 3056 if (texthole_arena[arena] == NULL) { 3057 ASSERT(texthole_source[arena] == NULL); 3058 3059 if (arena == 0) { 3060 texthole_source[0] = vmem_create("module_text_holesrc", 3061 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3062 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3063 0, VM_SLEEP); 3064 } else { 3065 base = HEAPTEXT_BASE + 3066 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3067 3068 (void) snprintf(c, sizeof (c), 3069 "heaptext_holesrc_%d", arena); 3070 3071 texthole_source[arena] = vmem_create(c, (void *)base, 3072 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3073 0, VM_SLEEP); 3074 } 3075 3076 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3077 3078 texthole_arena[arena] = vmem_create(c, NULL, 0, 3079 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3080 texthole_source[arena], 0, VM_SLEEP); 3081 } 3082 3083 mutex_exit(&texthole_lock); 3084 3085 ASSERT(texthole_arena[arena] != NULL); 3086 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3087 return (vmem_alloc(texthole_arena[arena], size, 3088 VM_BESTFIT | VM_NOSLEEP)); 3089 } 3090 3091 void 3092 kobj_texthole_free(caddr_t addr, size_t size) 3093 { 3094 int arena = HEAPTEXT_ARENA(addr); 3095 3096 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3097 ASSERT(texthole_arena[arena] != NULL); 3098 vmem_free(texthole_arena[arena], addr, size); 3099 } 3100