1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern int cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 74 extern void mach_dump_buffer_init(void); 75 extern void mach_descrip_init(void); 76 extern void mach_descrip_startup_fini(void); 77 extern void mach_memscrub(void); 78 extern void mach_fpras(void); 79 extern void mach_cpu_halt_idle(void); 80 extern void mach_hw_copy_limit(void); 81 extern void load_mach_drivers(void); 82 extern void load_tod_module(void); 83 #pragma weak load_tod_module 84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata); 86 #pragma weak ndata_alloc_mmfsa 87 88 extern void cif_init(void); 89 #pragma weak cif_init 90 91 extern void parse_idprom(void); 92 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 93 extern void mem_config_init(void); 94 extern void memseg_remap_init(void); 95 96 extern void mach_kpm_init(void); 97 extern int size_pse_array(pgcnt_t, int); 98 99 /* 100 * External Data: 101 */ 102 extern int vac_size; /* cache size in bytes */ 103 extern uint_t vac_mask; /* VAC alignment consistency mask */ 104 extern uint_t vac_colors; 105 106 /* 107 * Global Data Definitions: 108 */ 109 110 /* 111 * XXX - Don't port this to new architectures 112 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 113 * 'romp' has no use with a prom with an IEEE 1275 client interface. 114 * The driver doesn't use the value, but it depends on the symbol. 115 */ 116 void *romp; /* veritas driver won't load without romp 4154976 */ 117 /* 118 * Declare these as initialized data so we can patch them. 119 */ 120 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 121 pgcnt_t segkpsize = 122 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 123 uint_t segmap_percent = 12; /* Size of segmap segment */ 124 125 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 126 int vac_copyback = 1; 127 char *cache_mode = NULL; 128 int use_mix = 1; 129 int prom_debug = 0; 130 131 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 132 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 133 uint_t tba_taken_over = 0; 134 135 caddr_t s_text; /* start of kernel text segment */ 136 caddr_t e_text; /* end of kernel text segment */ 137 caddr_t s_data; /* start of kernel data segment */ 138 caddr_t e_data; /* end of kernel data segment */ 139 140 caddr_t modtext; /* beginning of module text */ 141 size_t modtext_sz; /* size of module text */ 142 caddr_t moddata; /* beginning of module data reserve */ 143 caddr_t e_moddata; /* end of module data reserve */ 144 145 /* 146 * End of first block of contiguous kernel in 32-bit virtual address space 147 */ 148 caddr_t econtig32; /* end of first blk of contiguous kernel */ 149 150 caddr_t ncbase; /* beginning of non-cached segment */ 151 caddr_t ncend; /* end of non-cached segment */ 152 caddr_t sdata; /* beginning of data segment */ 153 154 caddr_t extra_etva; /* beginning of unused nucleus text */ 155 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 156 157 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 158 caddr_t nalloc_base; /* beginning of nucleus allocation */ 159 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 160 caddr_t valloc_base; /* beginning of kvalloc segment */ 161 162 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 163 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 164 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 165 int kmem64_alignsize; /* page size for mem segment in 64-bit space */ 166 int kmem64_szc; /* page size code */ 167 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 168 169 uintptr_t shm_alignment; /* VAC address consistency modulus */ 170 struct memlist *phys_install; /* Total installed physical memory */ 171 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 172 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 173 struct memlist ndata; /* memlist of nucleus allocatable memory */ 174 int memexp_flag; /* memory expansion card flag */ 175 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 176 pgcnt_t obp_pages; /* Physical pages used by OBP */ 177 178 /* 179 * VM data structures 180 */ 181 long page_hashsz; /* Size of page hash table (power of two) */ 182 struct page *pp_base; /* Base of system page struct array */ 183 size_t pp_sz; /* Size in bytes of page struct array */ 184 struct page **page_hash; /* Page hash table */ 185 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 186 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 187 int pse_shift; /* log2(pse_table_size) */ 188 struct seg ktextseg; /* Segment used for kernel executable image */ 189 struct seg kvalloc; /* Segment used for "valloc" mapping */ 190 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 191 struct seg ktexthole; /* Segment used for nucleus text hole */ 192 struct seg kmapseg; /* Segment used for generic kernel mappings */ 193 struct seg kpmseg; /* Segment used for physical mapping */ 194 struct seg kdebugseg; /* Segment used for the kernel debugger */ 195 196 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 197 size_t kpm_pp_sz; /* Size of system kpm_page array */ 198 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 199 200 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 201 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 202 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 203 204 int segzio_fromheap = 0; /* zio allocations occur from heap */ 205 caddr_t segzio_base; /* Base address of segzio */ 206 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 207 208 /* 209 * debugger pages (if allocated) 210 */ 211 struct vnode kdebugvp; 212 213 /* 214 * VA range available to the debugger 215 */ 216 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 217 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 218 219 /* 220 * Segment for relocated kernel structures in 64-bit large RAM kernels 221 */ 222 struct seg kmem64; 223 224 struct memseg *memseg_base; 225 size_t memseg_sz; /* Used to translate a va to page */ 226 struct vnode unused_pages_vp; 227 228 /* 229 * VM data structures allocated early during boot. 230 */ 231 size_t pagehash_sz; 232 uint64_t memlist_sz; 233 234 char tbr_wr_addr_inited = 0; 235 236 caddr_t mpo_heap32_buf = NULL; 237 size_t mpo_heap32_bufsz = 0; 238 239 /* 240 * Static Routines: 241 */ 242 static void memlist_add(uint64_t, uint64_t, struct memlist **, 243 struct memlist **); 244 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 245 pgcnt_t); 246 static void kvm_init(void); 247 248 static void startup_init(void); 249 static void startup_memlist(void); 250 static void startup_modules(void); 251 static void startup_bop_gone(void); 252 static void startup_vm(void); 253 static void startup_end(void); 254 static void setup_cage_params(void); 255 static void startup_create_io_node(void); 256 257 static pgcnt_t npages; 258 static struct memlist *memlist; 259 void *memlist_end; 260 261 static pgcnt_t bop_alloc_pages; 262 static caddr_t hblk_base; 263 uint_t hblk_alloc_dynamic = 0; 264 uint_t hblk1_min = H1MIN; 265 266 267 /* 268 * Hooks for unsupported platforms and down-rev firmware 269 */ 270 int iam_positron(void); 271 #pragma weak iam_positron 272 static void do_prom_version_check(void); 273 static void kpm_init(void); 274 static void kpm_npages_setup(int); 275 static void kpm_memseg_init(void); 276 277 /* 278 * After receiving a thermal interrupt, this is the number of seconds 279 * to delay before shutting off the system, assuming 280 * shutdown fails. Use /etc/system to change the delay if this isn't 281 * large enough. 282 */ 283 int thermal_powerdown_delay = 1200; 284 285 /* 286 * Used to hold off page relocations into the cage until OBP has completed 287 * its boot-time handoff of its resources to the kernel. 288 */ 289 int page_relocate_ready = 0; 290 291 /* 292 * Enable some debugging messages concerning memory usage... 293 */ 294 #ifdef DEBUGGING_MEM 295 static int debugging_mem; 296 static void 297 printmemlist(char *title, struct memlist *list) 298 { 299 if (!debugging_mem) 300 return; 301 302 printf("%s\n", title); 303 304 while (list) { 305 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 306 (uint32_t)(list->address >> 32), (uint32_t)list->address, 307 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 308 list = list->next; 309 } 310 } 311 312 void 313 printmemseg(struct memseg *memseg) 314 { 315 if (!debugging_mem) 316 return; 317 318 printf("memseg\n"); 319 320 while (memseg) { 321 prom_printf("\tpage = 0x%p, epage = 0x%p, " 322 "pfn = 0x%x, epfn = 0x%x\n", 323 memseg->pages, memseg->epages, 324 memseg->pages_base, memseg->pages_end); 325 memseg = memseg->next; 326 } 327 } 328 329 #define debug_pause(str) halt((str)) 330 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 331 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 332 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 333 #define MPRINTF3(str, a, b, c) \ 334 if (debugging_mem) prom_printf((str), (a), (b), (c)) 335 #else /* DEBUGGING_MEM */ 336 #define MPRINTF(str) 337 #define MPRINTF1(str, a) 338 #define MPRINTF2(str, a, b) 339 #define MPRINTF3(str, a, b, c) 340 #endif /* DEBUGGING_MEM */ 341 342 /* Simple message to indicate that the bootops pointer has been zeroed */ 343 #ifdef DEBUG 344 static int bootops_gone_on = 0; 345 #define BOOTOPS_GONE() \ 346 if (bootops_gone_on) \ 347 prom_printf("The bootops vec is zeroed now!\n"); 348 #else 349 #define BOOTOPS_GONE() 350 #endif /* DEBUG */ 351 352 /* 353 * Monitor pages may not be where this says they are. 354 * and the debugger may not be there either. 355 * 356 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 357 * 358 * Physical memory layout 359 * (not necessarily contiguous) 360 * (THIS IS SOMEWHAT WRONG) 361 * /-----------------------\ 362 * | monitor pages | 363 * availmem -|-----------------------| 364 * | | 365 * | page pool | 366 * | | 367 * |-----------------------| 368 * | configured tables | 369 * | buffers | 370 * firstaddr -|-----------------------| 371 * | hat data structures | 372 * |-----------------------| 373 * | kernel data, bss | 374 * |-----------------------| 375 * | interrupt stack | 376 * |-----------------------| 377 * | kernel text (RO) | 378 * |-----------------------| 379 * | trap table (4k) | 380 * |-----------------------| 381 * page 1 | panicbuf | 382 * |-----------------------| 383 * page 0 | reclaimed | 384 * |_______________________| 385 * 386 * 387 * 388 * Kernel's Virtual Memory Layout. 389 * /-----------------------\ 390 * 0xFFFFFFFF.FFFFFFFF -| |- 391 * | OBP's virtual page | 392 * | tables | 393 * 0xFFFFFFFC.00000000 -|-----------------------|- 394 * : : 395 * : : 396 * -|-----------------------|- 397 * | segzio | (base and size vary) 398 * 0xFFFFFE00.00000000 -|-----------------------|- 399 * | | Ultrasparc I/II support 400 * | segkpm segment | up to 2TB of physical 401 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 402 * | | 403 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 404 * : : 405 * : : 406 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 407 * | | ^ 408 * | UltraSPARC I/II call | | 409 * | bug requires an extra | | 410 * | 4 GB of space between | | 411 * | hole and used RAM | | 412 * | | | 413 * 0xFFFFF800.00000000 -|-----------------------|- | 414 * | | | 415 * | Virtual Address Hole | UltraSPARC 416 * | on UltraSPARC I/II | I/II * ONLY * 417 * | | | 418 * 0x00000800.00000000 -|-----------------------|- | 419 * | | | 420 * | UltraSPARC I/II call | | 421 * | bug requires an extra | | 422 * | 4 GB of space between | | 423 * | hole and used RAM | | 424 * | | v 425 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 426 * : : ^ 427 * : : | 428 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 429 * | overmapped area | alignend_end | 430 * | (kmem64_alignsize | | 431 * | boundary) | | 432 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 433 * | | | 434 * | 64-bit kernel ONLY | | 435 * | | | 436 * | kmem64 segment | | 437 * | | | 438 * | (Relocated extra HME | Approximately 439 * | block allocations, | 1 TB of virtual 440 * | memnode freelists, | address space 441 * | HME hash buckets, | | 442 * | mml_table, kpmp_table,| | 443 * | page_t array and | | 444 * | hashblock pool to | | 445 * | avoid hard-coded | | 446 * | 32-bit vaddr | | 447 * | limitations) | | 448 * | | v 449 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 450 * | | 451 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 452 * | | 453 * 0x00000300.00000000 -|-----------------------|- SYSBASE 454 * : : 455 * : : 456 * -|-----------------------|- 457 * | | 458 * | segmap segment | SEGMAPSIZE (1/8th physmem, 459 * | | 256G MAX) 460 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 461 * : : 462 * : : 463 * -|-----------------------|- 464 * | | 465 * | segkp | SEGKPSIZE (2GB) 466 * | | 467 * | | 468 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 469 * | | 470 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 471 * | | (SEGKPBASE - 0x400000) 472 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 473 * | | (MEMSCRUBBASE - NCARGS) 474 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 475 * | | (ARGSBASE - PPMAPSIZE) 476 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 477 * | | 478 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 479 * | | 480 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 481 * : : 482 * : : 483 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 484 * | | 485 * | OBP | 486 * | | 487 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 488 * | kmdb | 489 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 490 * : : 491 * : : 492 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 493 * | | 494 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 495 * | | ~64MB) 496 * 0x00000000.78002000 -|-----------------------| 497 * | panicbuf | 498 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 499 * : : 500 * : : 501 * | | 502 * |-----------------------|- econtig32 503 * | vm structures | 504 * 0x00000000.01C00000 |-----------------------|- nalloc_end 505 * | TSBs | 506 * |-----------------------|- end/nalloc_base 507 * | kernel data & bss | 508 * 0x00000000.01800000 -|-----------------------| 509 * : nucleus text hole : 510 * 0x00000000.01400000 -|-----------------------| 511 * : : 512 * |-----------------------| 513 * | module text | 514 * |-----------------------|- e_text/modtext 515 * | kernel text | 516 * |-----------------------| 517 * | trap table (48k) | 518 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 519 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 520 * |-----------------------| 521 * | | 522 * | invalid | 523 * | | 524 * 0x00000000.00000000 _|_______________________| 525 * 526 * 527 * 528 * 32-bit User Virtual Memory Layout. 529 * /-----------------------\ 530 * | | 531 * | invalid | 532 * | | 533 * 0xFFC00000 -|-----------------------|- USERLIMIT 534 * | user stack | 535 * : : 536 * : : 537 * : : 538 * | user data | 539 * -|-----------------------|- 540 * | user text | 541 * 0x00002000 -|-----------------------|- 542 * | invalid | 543 * 0x00000000 _|_______________________| 544 * 545 * 546 * 547 * 64-bit User Virtual Memory Layout. 548 * /-----------------------\ 549 * | | 550 * | invalid | 551 * | | 552 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 553 * | user stack | 554 * : : 555 * : : 556 * : : 557 * | user data | 558 * -|-----------------------|- 559 * | user text | 560 * 0x00000000.00100000 -|-----------------------|- 561 * | invalid | 562 * 0x00000000.00000000 _|_______________________| 563 */ 564 565 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 566 extern uint64_t ecache_flush_address(void); 567 568 #pragma weak load_platform_modules 569 #pragma weak plat_startup_memlist 570 #pragma weak ecache_init_scrub_flush_area 571 #pragma weak ecache_flush_address 572 573 574 /* 575 * By default the DR Cage is enabled for maximum OS 576 * MPSS performance. Users needing to disable the cage mechanism 577 * can set this variable to zero via /etc/system. 578 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 579 * will result in loss of DR functionality. 580 * Platforms wishing to disable kernel Cage by default 581 * should do so in their set_platform_defaults() routine. 582 */ 583 int kernel_cage_enable = 1; 584 585 static void 586 setup_cage_params(void) 587 { 588 void (*func)(void); 589 590 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 591 if (func != NULL) { 592 (*func)(); 593 return; 594 } 595 596 if (kernel_cage_enable == 0) { 597 return; 598 } 599 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 600 601 if (kcage_on) { 602 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 603 } else { 604 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 605 } 606 607 } 608 609 /* 610 * Machine-dependent startup code 611 */ 612 void 613 startup(void) 614 { 615 startup_init(); 616 if (&startup_platform) 617 startup_platform(); 618 startup_memlist(); 619 startup_modules(); 620 setup_cage_params(); 621 startup_bop_gone(); 622 startup_vm(); 623 startup_end(); 624 } 625 626 struct regs sync_reg_buf; 627 uint64_t sync_tt; 628 629 void 630 sync_handler(void) 631 { 632 struct panic_trap_info ti; 633 int i; 634 635 /* 636 * Prevent trying to talk to the other CPUs since they are 637 * sitting in the prom and won't reply. 638 */ 639 for (i = 0; i < NCPU; i++) { 640 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 641 cpu[i]->cpu_flags &= ~CPU_READY; 642 cpu[i]->cpu_flags |= CPU_QUIESCED; 643 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 644 } 645 } 646 647 /* 648 * We've managed to get here without going through the 649 * normal panic code path. Try and save some useful 650 * information. 651 */ 652 if (!panicstr && (curthread->t_panic_trap == NULL)) { 653 ti.trap_type = sync_tt; 654 ti.trap_regs = &sync_reg_buf; 655 ti.trap_addr = NULL; 656 ti.trap_mmu_fsr = 0x0; 657 658 curthread->t_panic_trap = &ti; 659 } 660 661 /* 662 * If we're re-entering the panic path, update the signature 663 * block so that the SC knows we're in the second part of panic. 664 */ 665 if (panicstr) 666 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 667 668 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 669 panic("sync initiated"); 670 } 671 672 673 static void 674 startup_init(void) 675 { 676 /* 677 * We want to save the registers while we're still in OBP 678 * so that we know they haven't been fiddled with since. 679 * (In principle, OBP can't change them just because it 680 * makes a callback, but we'd rather not depend on that 681 * behavior.) 682 */ 683 char sync_str[] = 684 "warning @ warning off : sync " 685 "%%tl-c %%tstate h# %p x! " 686 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 687 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 688 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 689 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 690 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 691 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 692 "%%y h# %p l! %%tl-c %%tt h# %p x! " 693 "sync ; warning !"; 694 695 /* 696 * 20 == num of %p substrings 697 * 16 == max num of chars %p will expand to. 698 */ 699 char bp[sizeof (sync_str) + 16 * 20]; 700 701 (void) check_boot_version(BOP_GETVERSION(bootops)); 702 703 /* 704 * Initialize ptl1 stack for the 1st CPU. 705 */ 706 ptl1_init_cpu(&cpu0); 707 708 /* 709 * Initialize the address map for cache consistent mappings 710 * to random pages; must be done after vac_size is set. 711 */ 712 ppmapinit(); 713 714 /* 715 * Initialize the PROM callback handler. 716 */ 717 init_vx_handler(); 718 719 /* 720 * have prom call sync_callback() to handle the sync and 721 * save some useful information which will be stored in the 722 * core file later. 723 */ 724 (void) sprintf((char *)bp, sync_str, 725 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 726 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 727 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 728 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 729 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 730 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 731 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 732 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 733 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 734 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 735 prom_interpret(bp, 0, 0, 0, 0, 0); 736 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 737 } 738 739 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 740 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 741 742 #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 743 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 744 (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 745 746 #if !defined(C_OBP) 747 /* 748 * Install a temporary tte handler in OBP for kmem64 area. 749 * 750 * We map kmem64 area with large pages before the trap table is taken 751 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 752 * the same area. Duplicate tlb entries with different page sizes 753 * cause unpredicatble behavior. To avoid this, we don't create 754 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 755 * OBP). Instead, we manage translations with a temporary va>tte-data 756 * handler (kmem64-tte). This handler is replaced by unix-tte when 757 * the trap table is taken over. 758 * 759 * The temporary handler knows the physical address of the kmem64 760 * area. It uses the prom's pgmap@ Forth word for other addresses. 761 * 762 * We have to use BOP_ALLOC() method for C-OBP platforms because 763 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 764 * sun4u platforms. On sun4u we flush tlb after trap table is taken 765 * over if we use large pages for kernel heap and kmem64. Since sun4u 766 * prom (unlike sun4v) calls va>tte-data first for client address 767 * translation prom's ttes for kmem64 can't get into TLB even if we 768 * later switch to prom's trap table again. C-OBP uses 4M pages for 769 * client mappings when possible so on all platforms we get the 770 * benefit from large mappings for kmem64 area immediately during 771 * boot. 772 * 773 * pseudo code: 774 * if (context != 0) { 775 * return false 776 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 777 * tte = tte_template + 778 * (((miss_va & pagemask) - kmem64_base)); 779 * return tte, true 780 * } else { 781 * return pgmap@ result 782 * } 783 */ 784 char kmem64_obp_str[] = 785 "h# %lx constant kmem64_base " 786 "h# %lx constant kmem64_end " 787 "h# %lx constant kmem64_pagemask " 788 "h# %lx constant kmem64_template " 789 790 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 791 " if ( addr ) " 792 " drop false exit then ( false ) " 793 " dup kmem64_base kmem64_end within if ( addr ) " 794 " kmem64_pagemask and ( addr' ) " 795 " kmem64_base - ( addr' ) " 796 " kmem64_template + ( tte ) " 797 " true ( tte true ) " 798 " else ( addr ) " 799 " pgmap@ ( tte ) " 800 " dup 0< if true else drop false then ( tte true | false ) " 801 " then ( tte true | false ) " 802 "; " 803 804 "' kmem64-tte is va>tte-data " 805 ; 806 807 void 808 install_kmem64_tte() 809 { 810 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 811 tte_t tte; 812 813 PRM_DEBUG(kmem64_pabase); 814 PRM_DEBUG(kmem64_szc); 815 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 816 PROC_DATA | HAT_NOSYNC, kmem64_szc); 817 PRM_DEBUG(tte.ll); 818 (void) sprintf(b, kmem64_obp_str, 819 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 820 ASSERT(strlen(b) < sizeof (b)); 821 prom_interpret(b, 0, 0, 0, 0, 0); 822 } 823 #endif /* !C_OBP */ 824 825 /* 826 * As OBP takes up some RAM when the system boots, pages will already be "lost" 827 * to the system and reflected in npages by the time we see it. 828 * 829 * We only want to allocate kernel structures in the 64-bit virtual address 830 * space on systems with enough RAM to make the overhead of keeping track of 831 * an extra kernel memory segment worthwhile. 832 * 833 * Since OBP has already performed its memory allocations by this point, if we 834 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 835 * memory in the 64-bit virtual address space; otherwise keep allocations 836 * contiguous with we've mapped so far in the 32-bit virtual address space. 837 */ 838 #define MINMOVE_RAM_MB ((size_t)1900) 839 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 840 841 pgcnt_t tune_npages = (pgcnt_t) 842 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 843 844 #pragma weak page_set_colorequiv_arr_cpu 845 extern void page_set_colorequiv_arr_cpu(void); 846 847 static void 848 startup_memlist(void) 849 { 850 size_t alloc_sz; 851 size_t ctrs_sz; 852 caddr_t alloc_base; 853 caddr_t ctrs_base, ctrs_end; 854 caddr_t memspace; 855 caddr_t va; 856 int memblocks = 0; 857 struct memlist *cur; 858 size_t syslimit = (size_t)SYSLIMIT; 859 size_t sysbase = (size_t)SYSBASE; 860 int alloc_alignsize = ecache_alignsize; 861 int i; 862 extern void page_coloring_init(void); 863 extern void page_set_colorequiv_arr(void); 864 865 /* 866 * Initialize enough of the system to allow kmem_alloc to work by 867 * calling boot to allocate its memory until the time that 868 * kvm_init is completed. The page structs are allocated after 869 * rounding up end to the nearest page boundary; the memsegs are 870 * initialized and the space they use comes from the kernel heap. 871 * With appropriate initialization, they can be reallocated later 872 * to a size appropriate for the machine's configuration. 873 * 874 * At this point, memory is allocated for things that will never 875 * need to be freed, this used to be "valloced". This allows a 876 * savings as the pages don't need page structures to describe 877 * them because them will not be managed by the vm system. 878 */ 879 880 /* 881 * We're loaded by boot with the following configuration (as 882 * specified in the sun4u/conf/Mapfile): 883 * 884 * text: 4 MB chunk aligned on a 4MB boundary 885 * data & bss: 4 MB chunk aligned on a 4MB boundary 886 * 887 * These two chunks will eventually be mapped by 2 locked 4MB 888 * ttes and will represent the nucleus of the kernel. This gives 889 * us some free space that is already allocated, some or all of 890 * which is made available to kernel module text. 891 * 892 * The free space in the data-bss chunk is used for nucleus 893 * allocatable data structures and we reserve it using the 894 * nalloc_base and nalloc_end variables. This space is currently 895 * being used for hat data structures required for tlb miss 896 * handling operations. We align nalloc_base to a l2 cache 897 * linesize because this is the line size the hardware uses to 898 * maintain cache coherency. 899 * 256K is carved out for module data. 900 */ 901 902 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 903 moddata = nalloc_base; 904 e_moddata = nalloc_base + MODDATA; 905 nalloc_base = e_moddata; 906 907 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 908 valloc_base = nalloc_base; 909 910 /* 911 * Calculate the start of the data segment. 912 */ 913 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 914 915 PRM_DEBUG(moddata); 916 PRM_DEBUG(nalloc_base); 917 PRM_DEBUG(nalloc_end); 918 PRM_DEBUG(sdata); 919 920 /* 921 * Remember any slop after e_text so we can give it to the modules. 922 */ 923 PRM_DEBUG(e_text); 924 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 925 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 926 prom_panic("nucleus text overflow"); 927 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 928 modtext; 929 PRM_DEBUG(modtext); 930 PRM_DEBUG(modtext_sz); 931 932 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 933 &boot_physavail, &boot_physavail_len, 934 &boot_virtavail, &boot_virtavail_len); 935 /* 936 * Remember what the physically available highest page is 937 * so that dumpsys works properly, and find out how much 938 * memory is installed. 939 */ 940 installed_top_size_memlist_array(boot_physinstalled, 941 boot_physinstalled_len, &physmax, &physinstalled); 942 PRM_DEBUG(physinstalled); 943 PRM_DEBUG(physmax); 944 945 /* Fill out memory nodes config structure */ 946 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 947 948 /* 949 * Get the list of physically available memory to size 950 * the number of page structures needed. 951 */ 952 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 953 /* 954 * This first snap shot of npages can represent the pages used 955 * by OBP's text and data approximately. This is used in the 956 * the calculation of the kernel size 957 */ 958 obp_pages = physinstalled - npages; 959 960 961 /* 962 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 963 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 964 * 2MB) and any excess pages are put on physavail. The assumption is 965 * that small-memory systems will need more pages more than they'll 966 * need efficiently-mapped module texts. 967 */ 968 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 969 modtext_sz > MODTEXT_SM_CAP) { 970 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 971 modtext_sz = MODTEXT_SM_CAP; 972 extra_etva = modtext + modtext_sz; 973 } 974 975 PRM_DEBUG(extra_etpg); 976 PRM_DEBUG(modtext_sz); 977 PRM_DEBUG(extra_etva); 978 979 /* 980 * Account for any pages after e_text and e_data. 981 */ 982 npages += extra_etpg; 983 npages += mmu_btopr(nalloc_end - nalloc_base); 984 PRM_DEBUG(npages); 985 986 /* 987 * npages is the maximum of available physical memory possible. 988 * (ie. it will never be more than this) 989 */ 990 991 /* 992 * initialize the nucleus memory allocator. 993 */ 994 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 995 996 /* 997 * Allocate mmu fault status area from the nucleus data area. 998 */ 999 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1000 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1001 1002 /* 1003 * Allocate kernel TSBs from the nucleus data area. 1004 */ 1005 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1006 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1007 1008 /* 1009 * Allocate dmv dispatch table from the nucleus data area. 1010 */ 1011 if (ndata_alloc_dmv(&ndata) != 0) 1012 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1013 1014 1015 page_coloring_init(); 1016 1017 /* 1018 * Allocate page_freelists bin headers for memnode 0 from the 1019 * nucleus data area. 1020 */ 1021 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 1022 cmn_err(CE_PANIC, 1023 "no more nucleus memory after page free lists alloc"); 1024 1025 if (kpm_enable) { 1026 kpm_init(); 1027 /* 1028 * kpm page space -- Update kpm_npages and make the 1029 * same assumption about fragmenting as it is done 1030 * for memseg_sz. 1031 */ 1032 kpm_npages_setup(memblocks + 4); 1033 } 1034 1035 /* 1036 * Allocate hat related structs from the nucleus data area. 1037 */ 1038 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 1039 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1040 1041 /* 1042 * We want to do the BOP_ALLOCs before the real allocation of page 1043 * structs in order to not have to allocate page structs for this 1044 * memory. We need to calculate a virtual address because we want 1045 * the page structs to come before other allocations in virtual address 1046 * space. This is so some (if not all) of page structs can actually 1047 * live in the nucleus. 1048 */ 1049 1050 /* 1051 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1052 * 1053 * There are comments all over the SFMMU code warning of dire 1054 * consequences if the TSBs are moved out of 32-bit space. This 1055 * is largely because the asm code uses "sethi %hi(addr)"-type 1056 * instructions which will not provide the expected result if the 1057 * address is a 64-bit one. 1058 * 1059 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1060 */ 1061 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1062 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1063 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1064 PRM_DEBUG(alloc_base); 1065 1066 /* 1067 * Allocate IOMMU TSB array. We do this here so that the physical 1068 * memory gets deducted from the PROM's physical memory list. 1069 */ 1070 alloc_base = iommu_tsb_init(alloc_base); 1071 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1072 ecache_alignsize); 1073 PRM_DEBUG(alloc_base); 1074 1075 /* 1076 * Platforms like Starcat and OPL need special structures assigned in 1077 * 32-bit virtual address space because their probing routines execute 1078 * FCode, and FCode can't handle 64-bit virtual addresses... 1079 */ 1080 if (&plat_startup_memlist) { 1081 alloc_base = plat_startup_memlist(alloc_base); 1082 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1083 ecache_alignsize); 1084 PRM_DEBUG(alloc_base); 1085 } 1086 1087 /* 1088 * Save off where the contiguous allocations to date have ended 1089 * in econtig32. 1090 */ 1091 econtig32 = alloc_base; 1092 PRM_DEBUG(econtig32); 1093 1094 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1095 cmn_err(CE_PANIC, "econtig32 too big"); 1096 1097 /* 1098 * To avoid memory allocation collisions in the 32-bit virtual address 1099 * space, make allocations from this point forward in 64-bit virtual 1100 * address space starting at syslimit and working up. 1101 * 1102 * All this is needed because on large memory systems, the default 1103 * Solaris allocations will collide with SYSBASE32, which is hard 1104 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1105 * kernels, move the allocations to a location in the 64-bit virtual 1106 * address space space, allowing those structures to grow without 1107 * worry. 1108 * 1109 * On current CPUs we'll run out of physical memory address bits before 1110 * we need to worry about the allocations running into anything else in 1111 * VM or the virtual address holes on US-I and II, as there's currently 1112 * about 1 TB of addressable space before the US-I/II VA hole. 1113 */ 1114 kmem64_base = (caddr_t)syslimit; 1115 PRM_DEBUG(kmem64_base); 1116 1117 /* 1118 * Allocate addresses, but not physical memory. None of these locations 1119 * can be touched until physical memory is allocated below. 1120 */ 1121 alloc_base = kmem64_base; 1122 1123 /* 1124 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1125 * them here. 1126 */ 1127 if (khme_hash == NULL || uhme_hash == NULL) { 1128 /* 1129 * alloc_hme_buckets() will align alloc_base properly before 1130 * assigning the hash buckets, so we don't need to do it 1131 * before the call... 1132 */ 1133 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1134 1135 PRM_DEBUG(alloc_base); 1136 PRM_DEBUG(khme_hash); 1137 PRM_DEBUG(uhme_hash); 1138 } 1139 1140 /* 1141 * Allow for an early allocation of physically contiguous memory. 1142 */ 1143 alloc_base = contig_mem_prealloc(alloc_base, npages); 1144 1145 /* 1146 * Allocate the remaining page freelists. NUMA systems can 1147 * have lots of page freelists, one per node, which quickly 1148 * outgrow the amount of nucleus memory available. 1149 */ 1150 if (max_mem_nodes > 1) { 1151 int mnode; 1152 1153 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1154 alloc_base = alloc_page_freelists(mnode, alloc_base, 1155 ecache_alignsize); 1156 } 1157 PRM_DEBUG(alloc_base); 1158 } 1159 1160 if (!mml_table) { 1161 size_t mmltable_sz; 1162 1163 /* 1164 * We need to allocate the mml_table here because there 1165 * was not enough space within the nucleus. 1166 */ 1167 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1168 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1169 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1170 alloc_alignsize); 1171 mml_table = (kmutex_t *)alloc_base; 1172 alloc_base += alloc_sz; 1173 PRM_DEBUG(mml_table); 1174 PRM_DEBUG(alloc_base); 1175 } 1176 1177 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1178 size_t kpmptable_sz; 1179 caddr_t table; 1180 1181 /* 1182 * We need to allocate either kpmp_table or kpmp_stable here 1183 * because there was not enough space within the nucleus. 1184 */ 1185 kpmptable_sz = (kpm_smallpages == 0) ? 1186 sizeof (kpm_hlk_t) * kpmp_table_sz : 1187 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1188 1189 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1190 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1191 alloc_alignsize); 1192 1193 table = alloc_base; 1194 1195 if (kpm_smallpages == 0) { 1196 kpmp_table = (kpm_hlk_t *)table; 1197 PRM_DEBUG(kpmp_table); 1198 } else { 1199 kpmp_stable = (kpm_shlk_t *)table; 1200 PRM_DEBUG(kpmp_stable); 1201 } 1202 1203 alloc_base += alloc_sz; 1204 PRM_DEBUG(alloc_base); 1205 } 1206 1207 if (&ecache_init_scrub_flush_area) { 1208 /* 1209 * Pass alloc_base directly, as the routine itself is 1210 * responsible for any special alignment requirements... 1211 */ 1212 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1213 PRM_DEBUG(alloc_base); 1214 } 1215 1216 /* 1217 * Take the most current snapshot we can by calling mem-update. 1218 */ 1219 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1220 &boot_physavail, &boot_physavail_len, 1221 &boot_virtavail, &boot_virtavail_len); 1222 1223 /* 1224 * Reset npages and memblocks based on boot_physavail list. 1225 */ 1226 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1227 PRM_DEBUG(npages); 1228 1229 /* 1230 * Account for extra memory after e_text. 1231 */ 1232 npages += extra_etpg; 1233 1234 /* 1235 * Calculate the largest free memory chunk in the nucleus data area. 1236 * We need to figure out if page structs can fit in there or not. 1237 * We also make sure enough page structs get created for any physical 1238 * memory we might be returning to the system. 1239 */ 1240 ndata_remain_sz = ndata_maxsize(&ndata); 1241 PRM_DEBUG(ndata_remain_sz); 1242 1243 pp_sz = sizeof (struct page) * npages; 1244 1245 /* 1246 * Here's a nice bit of code based on somewhat recursive logic: 1247 * 1248 * If the page array would fit within the nucleus, we want to 1249 * add npages to cover any extra memory we may be returning back 1250 * to the system. 1251 * 1252 * HOWEVER, the page array is sized by calculating the size of 1253 * (struct page * npages), as are the pagehash table, ctrs and 1254 * memseg_list, so the very act of performing the calculation below may 1255 * in fact make the array large enough that it no longer fits in the 1256 * nucleus, meaning there would now be a much larger area of the 1257 * nucleus free that should really be added to npages, which would 1258 * make the page array that much larger, and so on. 1259 * 1260 * This also ignores the memory possibly used in the nucleus for the 1261 * the page hash, ctrs and memseg list and the fact that whether they 1262 * fit there or not varies with the npages calculation below, but we 1263 * don't even factor them into the equation at this point; perhaps we 1264 * should or perhaps we should just take the approach that the few 1265 * extra pages we could add via this calculation REALLY aren't worth 1266 * the hassle... 1267 */ 1268 if (ndata_remain_sz > pp_sz) { 1269 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1270 1271 npages += mmu_btop(spare); 1272 1273 pp_sz = npages * sizeof (struct page); 1274 1275 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1276 } 1277 1278 /* 1279 * If physmem is patched to be non-zero, use it instead of 1280 * the monitor value unless physmem is larger than the total 1281 * amount of memory on hand. 1282 */ 1283 if (physmem == 0 || physmem > npages) 1284 physmem = npages; 1285 1286 /* 1287 * If pp_base is NULL that means the routines above have determined 1288 * the page array will not fit in the nucleus; we'll have to 1289 * BOP_ALLOC() ourselves some space for them. 1290 */ 1291 if (pp_base == NULL) { 1292 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1293 alloc_alignsize); 1294 alloc_sz = roundup(pp_sz, alloc_alignsize); 1295 1296 pp_base = (struct page *)alloc_base; 1297 1298 alloc_base += alloc_sz; 1299 } 1300 1301 /* 1302 * The page structure hash table size is a power of 2 1303 * such that the average hash chain length is PAGE_HASHAVELEN. 1304 */ 1305 page_hashsz = npages / PAGE_HASHAVELEN; 1306 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1307 pagehash_sz = sizeof (struct page *) * page_hashsz; 1308 1309 /* 1310 * We want to TRY to fit the page structure hash table, 1311 * the page size free list counters, the memseg list and 1312 * and the kpm page space in the nucleus if possible. 1313 * 1314 * alloc_sz counts how much memory needs to be allocated by 1315 * BOP_ALLOC(). 1316 */ 1317 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1318 1319 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1320 1321 /* 1322 * Size up per page size free list counters. 1323 */ 1324 ctrs_sz = page_ctrs_sz(); 1325 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1326 1327 if (ctrs_base == NULL) 1328 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1329 1330 /* 1331 * The memseg list is for the chunks of physical memory that 1332 * will be managed by the vm system. The number calculated is 1333 * a guess as boot may fragment it more when memory allocations 1334 * are made before kphysm_init(). Currently, there are two 1335 * allocations before then, so we assume each causes fragmen- 1336 * tation, and add a couple more for good measure. 1337 */ 1338 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1339 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1340 1341 if (memseg_base == NULL) 1342 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1343 1344 1345 if (kpm_enable) { 1346 /* 1347 * kpm page space -- Update kpm_npages and make the 1348 * same assumption about fragmenting as it is done 1349 * for memseg_sz above. 1350 */ 1351 kpm_npages_setup(memblocks + 4); 1352 kpm_pp_sz = (kpm_smallpages == 0) ? 1353 kpm_npages * sizeof (kpm_page_t): 1354 kpm_npages * sizeof (kpm_spage_t); 1355 1356 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1357 ecache_alignsize); 1358 1359 if (kpm_pp_base == NULL) 1360 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1361 kpm_pp_sz; 1362 } 1363 1364 /* 1365 * Allocate the array that protects pp->p_selock. 1366 */ 1367 pse_shift = size_pse_array(physmem, max_ncpus); 1368 pse_table_size = 1 << pse_shift; 1369 pse_mutex = ndata_alloc(&ndata, pse_table_size * sizeof (pad_mutex_t), 1370 ecache_alignsize); 1371 if (pse_mutex == NULL) 1372 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1373 pse_table_size * sizeof (pad_mutex_t); 1374 1375 if (alloc_sz > 0) { 1376 uintptr_t bop_base; 1377 1378 /* 1379 * We need extra memory allocated through BOP_ALLOC. 1380 */ 1381 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1382 alloc_alignsize); 1383 1384 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1385 1386 bop_base = (uintptr_t)alloc_base; 1387 1388 alloc_base += alloc_sz; 1389 1390 if (page_hash == NULL) { 1391 page_hash = (struct page **)bop_base; 1392 bop_base = roundup(bop_base + pagehash_sz, 1393 ecache_alignsize); 1394 } 1395 1396 if (ctrs_base == NULL) { 1397 ctrs_base = (caddr_t)bop_base; 1398 bop_base = roundup(bop_base + ctrs_sz, 1399 ecache_alignsize); 1400 } 1401 1402 if (memseg_base == NULL) { 1403 memseg_base = (struct memseg *)bop_base; 1404 bop_base = roundup(bop_base + memseg_sz, 1405 ecache_alignsize); 1406 } 1407 1408 if (kpm_enable && kpm_pp_base == NULL) { 1409 kpm_pp_base = (uintptr_t)bop_base; 1410 bop_base = roundup(bop_base + kpm_pp_sz, 1411 ecache_alignsize); 1412 } 1413 1414 if (pse_mutex == NULL) { 1415 pse_mutex = (pad_mutex_t *)bop_base; 1416 bop_base = roundup(bop_base + 1417 pse_table_size * sizeof (pad_mutex_t), 1418 ecache_alignsize); 1419 } 1420 1421 ASSERT(bop_base <= (uintptr_t)alloc_base); 1422 } 1423 1424 PRM_DEBUG(page_hash); 1425 PRM_DEBUG(memseg_base); 1426 PRM_DEBUG(kpm_pp_base); 1427 PRM_DEBUG(kpm_pp_sz); 1428 PRM_DEBUG(pp_base); 1429 PRM_DEBUG(pp_sz); 1430 PRM_DEBUG(alloc_base); 1431 1432 #ifdef TRAPTRACE 1433 alloc_base = trap_trace_alloc(alloc_base); 1434 PRM_DEBUG(alloc_base); 1435 #endif /* TRAPTRACE */ 1436 1437 /* 1438 * In theory it's possible that kmem64 chunk is 0 sized 1439 * (on very small machines). Check for that. 1440 */ 1441 if (alloc_base == kmem64_base) { 1442 kmem64_base = NULL; 1443 kmem64_end = NULL; 1444 kmem64_aligned_end = NULL; 1445 goto kmem64_alloced; 1446 } 1447 1448 /* 1449 * Allocate kmem64 memory. 1450 * Round up to end of large page and overmap. 1451 * kmem64_end..kmem64_aligned_end is added to memory list for reuse 1452 */ 1453 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1454 MMU_PAGESIZE); 1455 1456 /* 1457 * Make one large memory alloc after figuring out the 64-bit size. This 1458 * will enable use of the largest page size appropriate for the system 1459 * architecture. 1460 */ 1461 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 1462 ASSERT(IS_P2ALIGNED(kmem64_base, TTEBYTES(max_bootlp_tteszc))); 1463 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 1464 size_t asize; 1465 #if !defined(C_OBP) 1466 unsigned long long pa; 1467 #endif /* !C_OBP */ 1468 1469 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 1470 continue; 1471 kmem64_alignsize = TTEBYTES(i); 1472 kmem64_szc = i; 1473 1474 /* limit page size for small memory */ 1475 if (mmu_btop(kmem64_alignsize) > (npages >> 2)) 1476 continue; 1477 1478 kmem64_aligned_end = (caddr_t)roundup((uintptr_t)kmem64_end, 1479 kmem64_alignsize); 1480 asize = kmem64_aligned_end - kmem64_base; 1481 #if !defined(C_OBP) 1482 if (prom_allocate_phys(asize, kmem64_alignsize, &pa) == 0) { 1483 if (prom_claim_virt(asize, kmem64_base) != 1484 (caddr_t)-1) { 1485 kmem64_pabase = pa; 1486 install_kmem64_tte(); 1487 break; 1488 } else { 1489 prom_free_phys(asize, pa); 1490 } 1491 } 1492 #else /* !C_OBP */ 1493 if ((caddr_t)BOP_ALLOC(bootops, kmem64_base, asize, 1494 kmem64_alignsize) == kmem64_base) { 1495 kmem64_pabase = va_to_pa(kmem64_base); 1496 break; 1497 } 1498 #endif /* !C_OBP */ 1499 if (i == TTE8K) { 1500 prom_panic("kmem64 allocation failure"); 1501 } 1502 } 1503 1504 PRM_DEBUG(kmem64_base); 1505 PRM_DEBUG(kmem64_end); 1506 PRM_DEBUG(kmem64_aligned_end); 1507 PRM_DEBUG(kmem64_alignsize); 1508 1509 /* 1510 * Now set pa using saved va from above. 1511 */ 1512 if (&ecache_init_scrub_flush_area) { 1513 (void) ecache_init_scrub_flush_area(NULL); 1514 } 1515 1516 kmem64_alloced: 1517 1518 /* 1519 * Initialize per page size free list counters. 1520 */ 1521 ctrs_end = page_ctrs_alloc(ctrs_base); 1522 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1523 1524 /* 1525 * Allocate space for the interrupt vector table and also for the 1526 * reserved interrupt vector data structures. 1527 */ 1528 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 1529 IVSIZE, MMU_PAGESIZE); 1530 if (memspace != (caddr_t)intr_vec_table) 1531 prom_panic("interrupt vector table allocation failure"); 1532 1533 /* 1534 * The memory lists from boot are allocated from the heap arena 1535 * so that later they can be freed and/or reallocated. 1536 */ 1537 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1538 prom_panic("could not retrieve property \"extent\""); 1539 1540 /* 1541 * Between now and when we finish copying in the memory lists, 1542 * allocations happen so the space gets fragmented and the 1543 * lists longer. Leave enough space for lists twice as long 1544 * as what boot says it has now; roundup to a pagesize. 1545 * Also add space for the final phys-avail copy in the fixup 1546 * routine. 1547 */ 1548 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1549 roundup(IVSIZE, MMU_PAGESIZE)); 1550 memlist_sz *= 4; 1551 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1552 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1553 if (memspace == NULL) 1554 halt("Boot allocation failed."); 1555 1556 memlist = (struct memlist *)memspace; 1557 memlist_end = (char *)memspace + memlist_sz; 1558 1559 PRM_DEBUG(memlist); 1560 PRM_DEBUG(memlist_end); 1561 PRM_DEBUG(sysbase); 1562 PRM_DEBUG(syslimit); 1563 1564 kernelheap_init((void *)sysbase, (void *)syslimit, 1565 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1566 1567 /* 1568 * Take the most current snapshot we can by calling mem-update. 1569 */ 1570 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1571 &boot_physavail, &boot_physavail_len, 1572 &boot_virtavail, &boot_virtavail_len); 1573 1574 /* 1575 * Remove the space used by BOP_ALLOC from the kernel heap 1576 * plus the area actually used by the OBP (if any) 1577 * ignoring virtual addresses in virt_avail, above syslimit. 1578 */ 1579 virt_avail = memlist; 1580 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1581 1582 for (cur = virt_avail; cur->next; cur = cur->next) { 1583 uint64_t range_base, range_size; 1584 1585 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1586 continue; 1587 if (range_base >= (uint64_t)syslimit) 1588 break; 1589 /* 1590 * Limit the range to end at syslimit. 1591 */ 1592 range_size = MIN(cur->next->address, 1593 (uint64_t)syslimit) - range_base; 1594 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1595 0, 0, (void *)range_base, (void *)(range_base + range_size), 1596 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1597 } 1598 1599 phys_avail = memlist; 1600 (void) copy_physavail(boot_physavail, boot_physavail_len, 1601 &memlist, 0, 0); 1602 1603 /* 1604 * Add any unused kmem64 memory from overmapped page 1605 * (Note: va_to_pa does not work for kmem64_end) 1606 */ 1607 if (kmem64_end < kmem64_aligned_end) { 1608 uint64_t overlap_size = kmem64_aligned_end - kmem64_end; 1609 uint64_t overlap_pa = kmem64_pabase + 1610 (kmem64_end - kmem64_base); 1611 1612 PRM_DEBUG(overlap_pa); 1613 PRM_DEBUG(overlap_size); 1614 memlist_add(overlap_pa, overlap_size, &memlist, &phys_avail); 1615 } 1616 1617 /* 1618 * Add any extra memory after e_text to the phys_avail list, as long 1619 * as there's at least a page to add. 1620 */ 1621 if (extra_etpg) 1622 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1623 &memlist, &phys_avail); 1624 1625 /* 1626 * Add any extra memory at the end of the ndata region if there's at 1627 * least a page to add. There might be a few more pages available in 1628 * the middle of the ndata region, but for now they are ignored. 1629 */ 1630 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1631 if (nalloc_base == NULL) 1632 nalloc_base = nalloc_end; 1633 ndata_remain_sz = nalloc_end - nalloc_base; 1634 1635 if (ndata_remain_sz >= MMU_PAGESIZE) 1636 memlist_add(va_to_pa(nalloc_base), 1637 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1638 1639 PRM_DEBUG(memlist); 1640 PRM_DEBUG(memlist_sz); 1641 PRM_DEBUG(memspace); 1642 1643 if ((caddr_t)memlist > (memspace + memlist_sz)) 1644 prom_panic("memlist overflow"); 1645 1646 PRM_DEBUG(pp_base); 1647 PRM_DEBUG(memseg_base); 1648 PRM_DEBUG(npages); 1649 1650 /* 1651 * Initialize the page structures from the memory lists. 1652 */ 1653 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1654 1655 availrmem_initial = availrmem = freemem; 1656 PRM_DEBUG(availrmem); 1657 1658 /* 1659 * Some of the locks depend on page_hashsz being set! 1660 * kmem_init() depends on this; so, keep it here. 1661 */ 1662 page_lock_init(); 1663 1664 /* 1665 * Initialize kernel memory allocator. 1666 */ 1667 kmem_init(); 1668 1669 /* 1670 * Factor in colorequiv to check additional 'equivalent' bins 1671 */ 1672 if (&page_set_colorequiv_arr_cpu != NULL) 1673 page_set_colorequiv_arr_cpu(); 1674 else 1675 page_set_colorequiv_arr(); 1676 1677 /* 1678 * Initialize bp_mapin(). 1679 */ 1680 bp_init(shm_alignment, HAT_STRICTORDER); 1681 1682 /* 1683 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1684 * vector data structures and MPO mblock structs from the 32-bit heap. 1685 */ 1686 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1687 panicbuf, panicbuf + PANICBUFSIZE, 1688 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1689 1690 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1691 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1692 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1693 1694 if (mpo_heap32_bufsz > (size_t)0) { 1695 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1696 PAGESIZE, 0, 0, mpo_heap32_buf, 1697 mpo_heap32_buf + mpo_heap32_bufsz, 1698 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1699 } 1700 mem_config_init(); 1701 } 1702 1703 static void 1704 startup_modules(void) 1705 { 1706 int proplen, nhblk1, nhblk8; 1707 size_t nhblksz; 1708 pgcnt_t pages_per_hblk; 1709 size_t hme8blk_sz, hme1blk_sz; 1710 1711 /* 1712 * Log any optional messages from the boot program 1713 */ 1714 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1715 if (proplen > 0) { 1716 char *msg; 1717 size_t len = (size_t)proplen; 1718 1719 msg = kmem_zalloc(len, KM_SLEEP); 1720 (void) BOP_GETPROP(bootops, "boot-message", msg); 1721 cmn_err(CE_CONT, "?%s\n", msg); 1722 kmem_free(msg, len); 1723 } 1724 1725 /* 1726 * Let the platforms have a chance to change default 1727 * values before reading system file. 1728 */ 1729 if (&set_platform_defaults) 1730 set_platform_defaults(); 1731 1732 /* 1733 * Calculate default settings of system parameters based upon 1734 * maxusers, yet allow to be overridden via the /etc/system file. 1735 */ 1736 param_calc(0); 1737 1738 mod_setup(); 1739 1740 /* 1741 * If this is a positron, complain and halt. 1742 */ 1743 if (&iam_positron && iam_positron()) { 1744 cmn_err(CE_WARN, "This hardware platform is not supported" 1745 " by this release of Solaris.\n"); 1746 #ifdef DEBUG 1747 prom_enter_mon(); /* Type 'go' to resume */ 1748 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1749 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1750 1751 #else /* DEBUG */ 1752 halt(0); 1753 #endif /* DEBUG */ 1754 } 1755 1756 /* 1757 * If we are running firmware that isn't 64-bit ready 1758 * then complain and halt. 1759 */ 1760 do_prom_version_check(); 1761 1762 /* 1763 * Initialize system parameters 1764 */ 1765 param_init(); 1766 1767 /* 1768 * maxmem is the amount of physical memory we're playing with. 1769 */ 1770 maxmem = physmem; 1771 1772 /* Set segkp limits. */ 1773 ncbase = kdi_segdebugbase; 1774 ncend = kdi_segdebugbase; 1775 1776 /* 1777 * Initialize the hat layer. 1778 */ 1779 hat_init(); 1780 1781 /* 1782 * Initialize segment management stuff. 1783 */ 1784 seg_init(); 1785 1786 /* 1787 * Create the va>tte handler, so the prom can understand 1788 * kernel translations. The handler is installed later, just 1789 * as we are about to take over the trap table from the prom. 1790 */ 1791 create_va_to_tte(); 1792 1793 /* 1794 * Load the forthdebugger (optional) 1795 */ 1796 forthdebug_init(); 1797 1798 /* 1799 * Create OBP node for console input callbacks 1800 * if it is needed. 1801 */ 1802 startup_create_io_node(); 1803 1804 if (modloadonly("fs", "specfs") == -1) 1805 halt("Can't load specfs"); 1806 1807 if (modloadonly("fs", "devfs") == -1) 1808 halt("Can't load devfs"); 1809 1810 if (modloadonly("misc", "swapgeneric") == -1) 1811 halt("Can't load swapgeneric"); 1812 1813 (void) modloadonly("sys", "lbl_edition"); 1814 1815 dispinit(); 1816 1817 /* 1818 * Infer meanings to the members of the idprom buffer. 1819 */ 1820 parse_idprom(); 1821 1822 /* Read cluster configuration data. */ 1823 clconf_init(); 1824 1825 setup_ddi(); 1826 1827 /* 1828 * Lets take this opportunity to load the root device. 1829 */ 1830 if (loadrootmodules() != 0) 1831 debug_enter("Can't load the root filesystem"); 1832 1833 /* 1834 * Load tod driver module for the tod part found on this system. 1835 * Recompute the cpu frequency/delays based on tod as tod part 1836 * tends to keep time more accurately. 1837 */ 1838 if (&load_tod_module) 1839 load_tod_module(); 1840 1841 /* 1842 * Allow platforms to load modules which might 1843 * be needed after bootops are gone. 1844 */ 1845 if (&load_platform_modules) 1846 load_platform_modules(); 1847 1848 setcpudelay(); 1849 1850 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1851 &boot_physavail, &boot_physavail_len, 1852 &boot_virtavail, &boot_virtavail_len); 1853 1854 /* 1855 * Calculation and allocation of hmeblks needed to remap 1856 * the memory allocated by PROM till now. 1857 * Overestimate the number of hblk1 elements by assuming 1858 * worst case of TTE64K mappings. 1859 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1860 */ 1861 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1862 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1863 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1864 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1865 1866 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1867 1868 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1869 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1870 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1871 1872 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1873 1874 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1875 nhblk8 = 0; 1876 while (bop_alloc_pages > 1) { 1877 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1878 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1879 bop_alloc_pages *= hme8blk_sz; 1880 bop_alloc_pages = btopr(bop_alloc_pages); 1881 } 1882 nhblk8 += 2; 1883 1884 /* 1885 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1886 * boundary, the number of hblk8's needed to map the entries in the 1887 * boot_virtavail list needs to be adjusted to take this into 1888 * consideration. Thus, we need to add additional hblk8's since it 1889 * is possible that an hblk8 will not have all 8 slots used due to 1890 * alignment constraints. Since there were boot_virtavail_len entries 1891 * in that list, we need to add that many hblk8's to the number 1892 * already calculated to make sure we don't underestimate. 1893 */ 1894 nhblk8 += boot_virtavail_len; 1895 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1896 1897 /* Allocate in pagesize chunks */ 1898 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1899 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1900 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1901 } 1902 1903 static void 1904 startup_bop_gone(void) 1905 { 1906 extern int bop_io_quiesced; 1907 1908 /* 1909 * Destroy the MD initialized at startup 1910 * The startup initializes the MD framework 1911 * using prom and BOP alloc free it now. 1912 */ 1913 mach_descrip_startup_fini(); 1914 1915 /* 1916 * Call back into boot and release boots resources. 1917 */ 1918 BOP_QUIESCE_IO(bootops); 1919 bop_io_quiesced = 1; 1920 1921 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1922 &boot_physavail, &boot_physavail_len, 1923 &boot_virtavail, &boot_virtavail_len); 1924 /* 1925 * Copy physinstalled list into kernel space. 1926 */ 1927 phys_install = memlist; 1928 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1929 1930 /* 1931 * setup physically contiguous area twice as large as the ecache. 1932 * this is used while doing displacement flush of ecaches 1933 */ 1934 if (&ecache_flush_address) { 1935 ecache_flushaddr = ecache_flush_address(); 1936 if (ecache_flushaddr == (uint64_t)-1) { 1937 cmn_err(CE_PANIC, 1938 "startup: no memory to set ecache_flushaddr"); 1939 } 1940 } 1941 1942 /* 1943 * Virtual available next. 1944 */ 1945 ASSERT(virt_avail != NULL); 1946 memlist_free_list(virt_avail); 1947 virt_avail = memlist; 1948 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1949 1950 /* 1951 * Last chance to ask our booter questions .. 1952 */ 1953 } 1954 1955 1956 /* 1957 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1958 * allocations have been performed. We can't call it in startup_bop_gone 1959 * since later operations can cause obp to allocate more memory. 1960 */ 1961 void 1962 startup_fixup_physavail(void) 1963 { 1964 struct memlist *cur; 1965 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1966 1967 PRM_DEBUG(kmem64_overmap_size); 1968 1969 /* 1970 * take the most current snapshot we can by calling mem-update 1971 */ 1972 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1973 &boot_physavail, &boot_physavail_len, 1974 &boot_virtavail, &boot_virtavail_len); 1975 1976 /* 1977 * Copy phys_avail list, again. 1978 * Both the kernel/boot and the prom have been allocating 1979 * from the original list we copied earlier. 1980 */ 1981 cur = memlist; 1982 (void) copy_physavail(boot_physavail, boot_physavail_len, 1983 &memlist, 0, 0); 1984 1985 /* 1986 * Add any unused kmem64 memory from overmapped page 1987 * (Note: va_to_pa does not work for kmem64_end) 1988 */ 1989 if (kmem64_overmap_size) { 1990 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1991 kmem64_overmap_size, 1992 &memlist, &cur); 1993 } 1994 1995 /* 1996 * Add any extra memory after e_text we added to the phys_avail list 1997 * back to the old list. 1998 */ 1999 if (extra_etpg) 2000 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 2001 &memlist, &cur); 2002 if (ndata_remain_sz >= MMU_PAGESIZE) 2003 memlist_add(va_to_pa(nalloc_base), 2004 (uint64_t)ndata_remain_sz, &memlist, &cur); 2005 2006 /* 2007 * There isn't any bounds checking on the memlist area 2008 * so ensure it hasn't overgrown. 2009 */ 2010 if ((caddr_t)memlist > (caddr_t)memlist_end) 2011 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 2012 2013 /* 2014 * The kernel removes the pages that were allocated for it from 2015 * the freelist, but we now have to find any -extra- pages that 2016 * the prom has allocated for it's own book-keeping, and remove 2017 * them from the freelist too. sigh. 2018 */ 2019 fix_prom_pages(phys_avail, cur); 2020 2021 ASSERT(phys_avail != NULL); 2022 memlist_free_list(phys_avail); 2023 phys_avail = cur; 2024 2025 /* 2026 * We're done with boot. Just after this point in time, boot 2027 * gets unmapped, so we can no longer rely on its services. 2028 * Zero the bootops to indicate this fact. 2029 */ 2030 bootops = (struct bootops *)NULL; 2031 BOOTOPS_GONE(); 2032 } 2033 2034 static void 2035 startup_vm(void) 2036 { 2037 size_t i; 2038 struct segmap_crargs a; 2039 struct segkpm_crargs b; 2040 2041 uint64_t avmem; 2042 caddr_t va; 2043 pgcnt_t max_phys_segkp; 2044 int mnode; 2045 2046 extern int use_brk_lpg, use_stk_lpg; 2047 2048 /* 2049 * get prom's mappings, create hments for them and switch 2050 * to the kernel context. 2051 */ 2052 hat_kern_setup(); 2053 2054 /* 2055 * Take over trap table 2056 */ 2057 setup_trap_table(); 2058 2059 /* 2060 * Install the va>tte handler, so that the prom can handle 2061 * misses and understand the kernel table layout in case 2062 * we need call into the prom. 2063 */ 2064 install_va_to_tte(); 2065 2066 /* 2067 * Set a flag to indicate that the tba has been taken over. 2068 */ 2069 tba_taken_over = 1; 2070 2071 /* initialize MMU primary context register */ 2072 mmu_init_kcontext(); 2073 2074 /* 2075 * The boot cpu can now take interrupts, x-calls, x-traps 2076 */ 2077 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 2078 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 2079 2080 /* 2081 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 2082 */ 2083 tbr_wr_addr_inited = 1; 2084 2085 /* 2086 * Initialize VM system, and map kernel address space. 2087 */ 2088 kvm_init(); 2089 2090 /* 2091 * XXX4U: previously, we initialized and turned on 2092 * the caches at this point. But of course we have 2093 * nothing to do, as the prom has already done this 2094 * for us -- main memory must be E$able at all times. 2095 */ 2096 2097 /* 2098 * If the following is true, someone has patched 2099 * phsymem to be less than the number of pages that 2100 * the system actually has. Remove pages until system 2101 * memory is limited to the requested amount. Since we 2102 * have allocated page structures for all pages, we 2103 * correct the amount of memory we want to remove 2104 * by the size of the memory used to hold page structures 2105 * for the non-used pages. 2106 */ 2107 if (physmem < npages) { 2108 pgcnt_t diff, off; 2109 struct page *pp; 2110 struct seg kseg; 2111 2112 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 2113 2114 off = 0; 2115 diff = npages - physmem; 2116 diff -= mmu_btopr(diff * sizeof (struct page)); 2117 kseg.s_as = &kas; 2118 while (diff--) { 2119 pp = page_create_va(&unused_pages_vp, (offset_t)off, 2120 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 2121 &kseg, (caddr_t)off); 2122 if (pp == NULL) 2123 cmn_err(CE_PANIC, "limited physmem too much!"); 2124 page_io_unlock(pp); 2125 page_downgrade(pp); 2126 availrmem--; 2127 off += MMU_PAGESIZE; 2128 } 2129 } 2130 2131 /* 2132 * When printing memory, show the total as physmem less 2133 * that stolen by a debugger. 2134 */ 2135 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 2136 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 2137 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 2138 2139 avmem = (uint64_t)freemem << PAGESHIFT; 2140 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 2141 2142 /* 2143 * For small memory systems disable automatic large pages. 2144 */ 2145 if (physmem < privm_lpg_min_physmem) { 2146 use_brk_lpg = 0; 2147 use_stk_lpg = 0; 2148 } 2149 2150 /* 2151 * Perform platform specific freelist processing 2152 */ 2153 if (&plat_freelist_process) { 2154 for (mnode = 0; mnode < max_mem_nodes; mnode++) 2155 if (mem_node_config[mnode].exists) 2156 plat_freelist_process(mnode); 2157 } 2158 2159 /* 2160 * Initialize the segkp segment type. We position it 2161 * after the configured tables and buffers (whose end 2162 * is given by econtig) and before V_WKBASE_ADDR. 2163 * Also in this area is segkmap (size SEGMAPSIZE). 2164 */ 2165 2166 /* XXX - cache alignment? */ 2167 va = (caddr_t)SEGKPBASE; 2168 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2169 2170 max_phys_segkp = (physmem * 2); 2171 2172 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2173 segkpsize = btop(SEGKPDEFSIZE); 2174 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2175 "segkpsize has been reset to %ld pages", segkpsize); 2176 } 2177 2178 i = ptob(MIN(segkpsize, max_phys_segkp)); 2179 2180 rw_enter(&kas.a_lock, RW_WRITER); 2181 if (seg_attach(&kas, va, i, segkp) < 0) 2182 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2183 if (segkp_create(segkp) != 0) 2184 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2185 rw_exit(&kas.a_lock); 2186 2187 /* 2188 * kpm segment 2189 */ 2190 segmap_kpm = kpm_enable && 2191 segmap_kpm && PAGESIZE == MAXBSIZE; 2192 2193 if (kpm_enable) { 2194 rw_enter(&kas.a_lock, RW_WRITER); 2195 2196 /* 2197 * The segkpm virtual range range is larger than the 2198 * actual physical memory size and also covers gaps in 2199 * the physical address range for the following reasons: 2200 * . keep conversion between segkpm and physical addresses 2201 * simple, cheap and unambiguous. 2202 * . avoid extension/shrink of the the segkpm in case of DR. 2203 * . avoid complexity for handling of virtual addressed 2204 * caches, segkpm and the regular mapping scheme must be 2205 * kept in sync wrt. the virtual color of mapped pages. 2206 * Any accesses to virtual segkpm ranges not backed by 2207 * physical memory will fall through the memseg pfn hash 2208 * and will be handled in segkpm_fault. 2209 * Additional kpm_size spaces needed for vac alias prevention. 2210 */ 2211 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2212 segkpm) < 0) 2213 cmn_err(CE_PANIC, "cannot attach segkpm"); 2214 2215 b.prot = PROT_READ | PROT_WRITE; 2216 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2217 2218 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2219 panic("segkpm_create segkpm"); 2220 2221 rw_exit(&kas.a_lock); 2222 2223 mach_kpm_init(); 2224 } 2225 2226 if (!segzio_fromheap) { 2227 size_t size; 2228 size_t physmem_b = mmu_ptob(physmem); 2229 2230 /* size is in bytes, segziosize is in pages */ 2231 if (segziosize == 0) { 2232 size = physmem_b; 2233 } else { 2234 size = mmu_ptob(segziosize); 2235 } 2236 2237 if (size < SEGZIOMINSIZE) { 2238 size = SEGZIOMINSIZE; 2239 } else if (size > SEGZIOMAXSIZE) { 2240 size = SEGZIOMAXSIZE; 2241 /* 2242 * On 64-bit x86, we only have 2TB of KVA. This exists 2243 * for parity with x86. 2244 * 2245 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2246 * doesn't consume all of KVA. However, if we have a 2247 * system that has more thant 512gb of physical memory, 2248 * we can actually consume about half of the difference 2249 * between 512gb and the rest of the available physical 2250 * memory. 2251 */ 2252 if (physmem_b > SEGZIOMAXSIZE) { 2253 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2254 } 2255 } 2256 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2257 /* put the base of the ZIO segment after the kpm segment */ 2258 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2259 PRM_DEBUG(segziosize); 2260 PRM_DEBUG(segzio_base); 2261 2262 /* 2263 * On some platforms, kvm_init is called after the kpm 2264 * sizes have been determined. On SPARC, kvm_init is called 2265 * before, so we have to attach the kzioseg after kvm is 2266 * initialized, otherwise we'll try to allocate from the boot 2267 * area since the kernel heap hasn't yet been configured. 2268 */ 2269 rw_enter(&kas.a_lock, RW_WRITER); 2270 2271 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2272 &kzioseg); 2273 (void) segkmem_zio_create(&kzioseg); 2274 2275 /* create zio area covering new segment */ 2276 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2277 2278 rw_exit(&kas.a_lock); 2279 } 2280 2281 2282 /* 2283 * Now create generic mapping segment. This mapping 2284 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2285 * virtual address is greater than the amount of free 2286 * memory that is available, then we trim back the 2287 * segment size to that amount 2288 */ 2289 va = (caddr_t)SEGMAPBASE; 2290 2291 /* 2292 * 1201049: segkmap base address must be MAXBSIZE aligned 2293 */ 2294 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2295 2296 /* 2297 * Set size of segmap to percentage of freemem at boot, 2298 * but stay within the allowable range 2299 * Note we take percentage before converting from pages 2300 * to bytes to avoid an overflow on 32-bit kernels. 2301 */ 2302 i = mmu_ptob((freemem * segmap_percent) / 100); 2303 2304 if (i < MINMAPSIZE) 2305 i = MINMAPSIZE; 2306 2307 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2308 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2309 2310 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2311 2312 rw_enter(&kas.a_lock, RW_WRITER); 2313 if (seg_attach(&kas, va, i, segkmap) < 0) 2314 cmn_err(CE_PANIC, "cannot attach segkmap"); 2315 2316 a.prot = PROT_READ | PROT_WRITE; 2317 a.shmsize = shm_alignment; 2318 a.nfreelist = 0; /* use segmap driver defaults */ 2319 2320 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2321 panic("segmap_create segkmap"); 2322 rw_exit(&kas.a_lock); 2323 2324 segdev_init(); 2325 } 2326 2327 static void 2328 startup_end(void) 2329 { 2330 if ((caddr_t)memlist > (caddr_t)memlist_end) 2331 panic("memlist overflow 2"); 2332 memlist_free_block((caddr_t)memlist, 2333 ((caddr_t)memlist_end - (caddr_t)memlist)); 2334 memlist = NULL; 2335 2336 /* enable page_relocation since OBP is now done */ 2337 page_relocate_ready = 1; 2338 2339 /* 2340 * Perform tasks that get done after most of the VM 2341 * initialization has been done but before the clock 2342 * and other devices get started. 2343 */ 2344 kern_setup1(); 2345 2346 /* 2347 * Intialize the VM arenas for allocating physically 2348 * contiguus memory chunk for interrupt queues snd 2349 * allocate/register boot cpu's queues, if any and 2350 * allocate dump buffer for sun4v systems to store 2351 * extra crash information during crash dump 2352 */ 2353 contig_mem_init(); 2354 mach_descrip_init(); 2355 2356 if (cpu_intrq_setup(CPU)) { 2357 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2358 } 2359 cpu_intrq_register(CPU); 2360 mach_htraptrace_setup(CPU->cpu_id); 2361 mach_htraptrace_configure(CPU->cpu_id); 2362 mach_dump_buffer_init(); 2363 2364 /* 2365 * Initialize interrupt related stuff 2366 */ 2367 cpu_intr_alloc(CPU, NINTR_THREADS); 2368 2369 (void) splzs(); /* allow hi clock ints but not zs */ 2370 2371 /* 2372 * Initialize errors. 2373 */ 2374 error_init(); 2375 2376 /* 2377 * Note that we may have already used kernel bcopy before this 2378 * point - but if you really care about this, adb the use_hw_* 2379 * variables to 0 before rebooting. 2380 */ 2381 mach_hw_copy_limit(); 2382 2383 /* 2384 * Install the "real" preemption guards before DDI services 2385 * are available. 2386 */ 2387 (void) prom_set_preprom(kern_preprom); 2388 (void) prom_set_postprom(kern_postprom); 2389 CPU->cpu_m.mutex_ready = 1; 2390 2391 /* 2392 * Initialize segnf (kernel support for non-faulting loads). 2393 */ 2394 segnf_init(); 2395 2396 /* 2397 * Configure the root devinfo node. 2398 */ 2399 configure(); /* set up devices */ 2400 mach_cpu_halt_idle(); 2401 } 2402 2403 2404 void 2405 post_startup(void) 2406 { 2407 #ifdef PTL1_PANIC_DEBUG 2408 extern void init_ptl1_thread(void); 2409 #endif /* PTL1_PANIC_DEBUG */ 2410 extern void abort_sequence_init(void); 2411 2412 /* 2413 * Set the system wide, processor-specific flags to be passed 2414 * to userland via the aux vector for performance hints and 2415 * instruction set extensions. 2416 */ 2417 bind_hwcap(); 2418 2419 /* 2420 * Startup memory scrubber (if any) 2421 */ 2422 mach_memscrub(); 2423 2424 /* 2425 * Allocate soft interrupt to handle abort sequence. 2426 */ 2427 abort_sequence_init(); 2428 2429 /* 2430 * Configure the rest of the system. 2431 * Perform forceloading tasks for /etc/system. 2432 */ 2433 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2434 /* 2435 * ON4.0: Force /proc module in until clock interrupt handle fixed 2436 * ON4.0: This must be fixed or restated in /etc/systems. 2437 */ 2438 (void) modload("fs", "procfs"); 2439 2440 /* load machine class specific drivers */ 2441 load_mach_drivers(); 2442 2443 /* load platform specific drivers */ 2444 if (&load_platform_drivers) 2445 load_platform_drivers(); 2446 2447 /* load vis simulation module, if we are running w/fpu off */ 2448 if (!fpu_exists) { 2449 if (modload("misc", "vis") == -1) 2450 halt("Can't load vis"); 2451 } 2452 2453 mach_fpras(); 2454 2455 maxmem = freemem; 2456 2457 #ifdef PTL1_PANIC_DEBUG 2458 init_ptl1_thread(); 2459 #endif /* PTL1_PANIC_DEBUG */ 2460 2461 if (&cif_init) 2462 cif_init(); 2463 } 2464 2465 #ifdef PTL1_PANIC_DEBUG 2466 int ptl1_panic_test = 0; 2467 int ptl1_panic_xc_one_test = 0; 2468 int ptl1_panic_xc_all_test = 0; 2469 int ptl1_panic_xt_one_test = 0; 2470 int ptl1_panic_xt_all_test = 0; 2471 kthread_id_t ptl1_thread_p = NULL; 2472 kcondvar_t ptl1_cv; 2473 kmutex_t ptl1_mutex; 2474 int ptl1_recurse_count_threshold = 0x40; 2475 int ptl1_recurse_trap_threshold = 0x3d; 2476 extern void ptl1_recurse(int, int); 2477 extern void ptl1_panic_xt(int, int); 2478 2479 /* 2480 * Called once per second by timeout() to wake up 2481 * the ptl1_panic thread to see if it should cause 2482 * a trap to the ptl1_panic() code. 2483 */ 2484 /* ARGSUSED */ 2485 static void 2486 ptl1_wakeup(void *arg) 2487 { 2488 mutex_enter(&ptl1_mutex); 2489 cv_signal(&ptl1_cv); 2490 mutex_exit(&ptl1_mutex); 2491 } 2492 2493 /* 2494 * ptl1_panic cross call function: 2495 * Needed because xc_one() and xc_some() can pass 2496 * 64 bit args but ptl1_recurse() expects ints. 2497 */ 2498 static void 2499 ptl1_panic_xc(void) 2500 { 2501 ptl1_recurse(ptl1_recurse_count_threshold, 2502 ptl1_recurse_trap_threshold); 2503 } 2504 2505 /* 2506 * The ptl1 thread waits for a global flag to be set 2507 * and uses the recurse thresholds to set the stack depth 2508 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2509 * or indirectly via the cross call and cross trap functions. 2510 * 2511 * This is useful testing stack overflows and normal 2512 * ptl1_panic() states with a know stack frame. 2513 * 2514 * ptl1_recurse() is an asm function in ptl1_panic.s that 2515 * sets the {In, Local, Out, and Global} registers to a 2516 * know state on the stack and just prior to causing a 2517 * test ptl1_panic trap. 2518 */ 2519 static void 2520 ptl1_thread(void) 2521 { 2522 mutex_enter(&ptl1_mutex); 2523 while (ptl1_thread_p) { 2524 cpuset_t other_cpus; 2525 int cpu_id; 2526 int my_cpu_id; 2527 int target_cpu_id; 2528 int target_found; 2529 2530 if (ptl1_panic_test) { 2531 ptl1_recurse(ptl1_recurse_count_threshold, 2532 ptl1_recurse_trap_threshold); 2533 } 2534 2535 /* 2536 * Find potential targets for x-call and x-trap, 2537 * if any exist while preempt is disabled we 2538 * start a ptl1_panic if requested via a 2539 * globals. 2540 */ 2541 kpreempt_disable(); 2542 my_cpu_id = CPU->cpu_id; 2543 other_cpus = cpu_ready_set; 2544 CPUSET_DEL(other_cpus, CPU->cpu_id); 2545 target_found = 0; 2546 if (!CPUSET_ISNULL(other_cpus)) { 2547 /* 2548 * Pick the first one 2549 */ 2550 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2551 if (cpu_id == my_cpu_id) 2552 continue; 2553 2554 if (CPU_XCALL_READY(cpu_id)) { 2555 target_cpu_id = cpu_id; 2556 target_found = 1; 2557 break; 2558 } 2559 } 2560 ASSERT(target_found); 2561 2562 if (ptl1_panic_xc_one_test) { 2563 xc_one(target_cpu_id, 2564 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2565 } 2566 if (ptl1_panic_xc_all_test) { 2567 xc_some(other_cpus, 2568 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2569 } 2570 if (ptl1_panic_xt_one_test) { 2571 xt_one(target_cpu_id, 2572 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2573 } 2574 if (ptl1_panic_xt_all_test) { 2575 xt_some(other_cpus, 2576 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2577 } 2578 } 2579 kpreempt_enable(); 2580 (void) timeout(ptl1_wakeup, NULL, hz); 2581 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2582 } 2583 mutex_exit(&ptl1_mutex); 2584 } 2585 2586 /* 2587 * Called during early startup to create the ptl1_thread 2588 */ 2589 void 2590 init_ptl1_thread(void) 2591 { 2592 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2593 &p0, TS_RUN, 0); 2594 } 2595 #endif /* PTL1_PANIC_DEBUG */ 2596 2597 2598 /* 2599 * Add to a memory list. 2600 * start = start of new memory segment 2601 * len = length of new memory segment in bytes 2602 * memlistp = pointer to array of available memory segment structures 2603 * curmemlistp = memory list to which to add segment. 2604 */ 2605 static void 2606 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2607 struct memlist **curmemlistp) 2608 { 2609 struct memlist *new; 2610 2611 new = *memlistp; 2612 new->address = start; 2613 new->size = len; 2614 *memlistp = new + 1; 2615 2616 memlist_insert(new, curmemlistp); 2617 } 2618 2619 /* 2620 * In the case of architectures that support dynamic addition of 2621 * memory at run-time there are two cases where memsegs need to 2622 * be initialized and added to the memseg list. 2623 * 1) memsegs that are constructed at startup. 2624 * 2) memsegs that are constructed at run-time on 2625 * hot-plug capable architectures. 2626 * This code was originally part of the function kphysm_init(). 2627 */ 2628 2629 static void 2630 memseg_list_add(struct memseg *memsegp) 2631 { 2632 struct memseg **prev_memsegp; 2633 pgcnt_t num; 2634 2635 /* insert in memseg list, decreasing number of pages order */ 2636 2637 num = MSEG_NPAGES(memsegp); 2638 2639 for (prev_memsegp = &memsegs; *prev_memsegp; 2640 prev_memsegp = &((*prev_memsegp)->next)) { 2641 if (num > MSEG_NPAGES(*prev_memsegp)) 2642 break; 2643 } 2644 2645 memsegp->next = *prev_memsegp; 2646 *prev_memsegp = memsegp; 2647 2648 if (kpm_enable) { 2649 memsegp->nextpa = (memsegp->next) ? 2650 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2651 2652 if (prev_memsegp != &memsegs) { 2653 struct memseg *msp; 2654 msp = (struct memseg *)((caddr_t)prev_memsegp - 2655 offsetof(struct memseg, next)); 2656 msp->nextpa = va_to_pa(memsegp); 2657 } else { 2658 memsegspa = va_to_pa(memsegs); 2659 } 2660 } 2661 } 2662 2663 /* 2664 * PSM add_physmem_cb(). US-II and newer processors have some 2665 * flavor of the prefetch capability implemented. We exploit 2666 * this capability for optimum performance. 2667 */ 2668 #define PREFETCH_BYTES 64 2669 2670 void 2671 add_physmem_cb(page_t *pp, pfn_t pnum) 2672 { 2673 extern void prefetch_page_w(void *); 2674 2675 pp->p_pagenum = pnum; 2676 2677 /* 2678 * Prefetch one more page_t into E$. To prevent future 2679 * mishaps with the sizeof(page_t) changing on us, we 2680 * catch this on debug kernels if we can't bring in the 2681 * entire hpage with 2 PREFETCH_BYTES reads. See 2682 * also, sun4u/cpu/cpu_module.c 2683 */ 2684 /*LINTED*/ 2685 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2686 prefetch_page_w((char *)pp); 2687 } 2688 2689 /* 2690 * kphysm_init() tackles the problem of initializing physical memory. 2691 * The old startup made some assumptions about the kernel living in 2692 * physically contiguous space which is no longer valid. 2693 */ 2694 static void 2695 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2696 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2697 { 2698 struct memlist *pmem; 2699 struct memseg *msp; 2700 pfn_t base; 2701 pgcnt_t num; 2702 pfn_t lastseg_pages_end = 0; 2703 pgcnt_t nelem_used = 0; 2704 2705 ASSERT(page_hash != NULL && page_hashsz != 0); 2706 2707 msp = memsegp; 2708 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2709 2710 /* 2711 * Build the memsegs entry 2712 */ 2713 num = btop(pmem->size); 2714 if (num > npages) 2715 num = npages; 2716 npages -= num; 2717 base = btop(pmem->address); 2718 2719 msp->pages = pp; 2720 msp->epages = pp + num; 2721 msp->pages_base = base; 2722 msp->pages_end = base + num; 2723 2724 if (kpm_enable) { 2725 pfn_t pbase_a; 2726 pfn_t pend_a; 2727 pfn_t prev_pend_a; 2728 pgcnt_t nelem; 2729 2730 msp->pagespa = va_to_pa(pp); 2731 msp->epagespa = va_to_pa(pp + num); 2732 pbase_a = kpmptop(ptokpmp(base)); 2733 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2734 nelem = ptokpmp(pend_a - pbase_a); 2735 msp->kpm_nkpmpgs = nelem; 2736 msp->kpm_pbase = pbase_a; 2737 if (lastseg_pages_end) { 2738 /* 2739 * Assume phys_avail is in ascending order 2740 * of physical addresses. 2741 */ 2742 ASSERT(base + num > lastseg_pages_end); 2743 prev_pend_a = kpmptop( 2744 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2745 2746 if (prev_pend_a > pbase_a) { 2747 /* 2748 * Overlap, more than one memseg may 2749 * point to the same kpm_page range. 2750 */ 2751 if (kpm_smallpages == 0) { 2752 msp->kpm_pages = 2753 (kpm_page_t *)kpm_pp - 1; 2754 kpm_pp = (uintptr_t) 2755 ((kpm_page_t *)kpm_pp 2756 + nelem - 1); 2757 } else { 2758 msp->kpm_spages = 2759 (kpm_spage_t *)kpm_pp - 1; 2760 kpm_pp = (uintptr_t) 2761 ((kpm_spage_t *)kpm_pp 2762 + nelem - 1); 2763 } 2764 nelem_used += nelem - 1; 2765 2766 } else { 2767 if (kpm_smallpages == 0) { 2768 msp->kpm_pages = 2769 (kpm_page_t *)kpm_pp; 2770 kpm_pp = (uintptr_t) 2771 ((kpm_page_t *)kpm_pp 2772 + nelem); 2773 } else { 2774 msp->kpm_spages = 2775 (kpm_spage_t *)kpm_pp; 2776 kpm_pp = (uintptr_t) 2777 ((kpm_spage_t *) 2778 kpm_pp + nelem); 2779 } 2780 nelem_used += nelem; 2781 } 2782 2783 } else { 2784 if (kpm_smallpages == 0) { 2785 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2786 kpm_pp = (uintptr_t) 2787 ((kpm_page_t *)kpm_pp + nelem); 2788 } else { 2789 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2790 kpm_pp = (uintptr_t) 2791 ((kpm_spage_t *)kpm_pp + nelem); 2792 } 2793 nelem_used = nelem; 2794 } 2795 2796 if (nelem_used > kpm_npages) 2797 panic("kphysm_init: kpm_pp overflow\n"); 2798 2799 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2800 lastseg_pages_end = msp->pages_end; 2801 } 2802 2803 memseg_list_add(msp); 2804 2805 /* 2806 * add_physmem() initializes the PSM part of the page 2807 * struct by calling the PSM back with add_physmem_cb(). 2808 * In addition it coalesces pages into larger pages as 2809 * it initializes them. 2810 */ 2811 add_physmem(pp, num, base); 2812 pp += num; 2813 msp++; 2814 } 2815 2816 build_pfn_hash(); 2817 } 2818 2819 /* 2820 * Kernel VM initialization. 2821 * Assumptions about kernel address space ordering: 2822 * (1) gap (user space) 2823 * (2) kernel text 2824 * (3) kernel data/bss 2825 * (4) gap 2826 * (5) kernel data structures 2827 * (6) gap 2828 * (7) debugger (optional) 2829 * (8) monitor 2830 * (9) gap (possibly null) 2831 * (10) dvma 2832 * (11) devices 2833 */ 2834 static void 2835 kvm_init(void) 2836 { 2837 /* 2838 * Put the kernel segments in kernel address space. 2839 */ 2840 rw_enter(&kas.a_lock, RW_WRITER); 2841 as_avlinit(&kas); 2842 2843 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2844 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2845 (void) segkmem_create(&ktextseg); 2846 2847 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2848 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2849 (void) segkmem_create(&ktexthole); 2850 2851 (void) seg_attach(&kas, (caddr_t)valloc_base, 2852 (size_t)(econtig32 - valloc_base), &kvalloc); 2853 (void) segkmem_create(&kvalloc); 2854 2855 if (kmem64_base) { 2856 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2857 (size_t)(kmem64_end - kmem64_base), &kmem64); 2858 (void) segkmem_create(&kmem64); 2859 } 2860 2861 /* 2862 * We're about to map out /boot. This is the beginning of the 2863 * system resource management transition. We can no longer 2864 * call into /boot for I/O or memory allocations. 2865 */ 2866 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2867 (void) segkmem_create(&kvseg); 2868 hblk_alloc_dynamic = 1; 2869 2870 /* 2871 * we need to preallocate pages for DR operations before enabling large 2872 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2873 */ 2874 memseg_remap_init(); 2875 2876 /* at this point we are ready to use large page heap */ 2877 segkmem_heap_lp_init(); 2878 2879 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2880 &kvseg32); 2881 (void) segkmem_create(&kvseg32); 2882 2883 /* 2884 * Create a segment for the debugger. 2885 */ 2886 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2887 (void) segkmem_create(&kdebugseg); 2888 2889 rw_exit(&kas.a_lock); 2890 } 2891 2892 char obp_tte_str[] = 2893 "h# %x constant MMU_PAGESHIFT " 2894 "h# %x constant TTE8K " 2895 "h# %x constant SFHME_SIZE " 2896 "h# %x constant SFHME_TTE " 2897 "h# %x constant HMEBLK_TAG " 2898 "h# %x constant HMEBLK_NEXT " 2899 "h# %x constant HMEBLK_MISC " 2900 "h# %x constant HMEBLK_HME1 " 2901 "h# %x constant NHMENTS " 2902 "h# %x constant HBLK_SZMASK " 2903 "h# %x constant HBLK_RANGE_SHIFT " 2904 "h# %x constant HMEBP_HBLK " 2905 "h# %x constant HMEBUCKET_SIZE " 2906 "h# %x constant HTAG_SFMMUPSZ " 2907 "h# %x constant HTAG_BSPAGE_SHIFT " 2908 "h# %x constant HTAG_REHASH_SHIFT " 2909 "h# %x constant SFMMU_INVALID_SHMERID " 2910 "h# %x constant mmu_hashcnt " 2911 "h# %p constant uhme_hash " 2912 "h# %p constant khme_hash " 2913 "h# %x constant UHMEHASH_SZ " 2914 "h# %x constant KHMEHASH_SZ " 2915 "h# %p constant KCONTEXT " 2916 "h# %p constant KHATID " 2917 "h# %x constant ASI_MEM " 2918 2919 ": PHYS-X@ ( phys -- data ) " 2920 " ASI_MEM spacex@ " 2921 "; " 2922 2923 ": PHYS-W@ ( phys -- data ) " 2924 " ASI_MEM spacew@ " 2925 "; " 2926 2927 ": PHYS-L@ ( phys -- data ) " 2928 " ASI_MEM spaceL@ " 2929 "; " 2930 2931 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2932 " 3 * MMU_PAGESHIFT + " 2933 "; " 2934 2935 ": TTE_IS_VALID ( ttep -- flag ) " 2936 " PHYS-X@ 0< " 2937 "; " 2938 2939 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2940 " dup TTE8K = if " 2941 " drop HBLK_RANGE_SHIFT " 2942 " else " 2943 " TTE_PAGE_SHIFT " 2944 " then " 2945 "; " 2946 2947 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2948 " tuck >> swap MMU_PAGESHIFT - << " 2949 "; " 2950 2951 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2952 " >> over xor swap ( hash sfmmup ) " 2953 " KHATID <> if ( hash ) " 2954 " UHMEHASH_SZ and ( bucket ) " 2955 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2956 " else ( hash ) " 2957 " KHMEHASH_SZ and ( bucket ) " 2958 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2959 " then ( hmebp ) " 2960 "; " 2961 2962 ": HME_HASH_TABLE_SEARCH " 2963 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2964 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2965 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2966 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2967 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2968 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2969 " else " 2970 " hmeblk_next + phys-x@ false " 2971 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2972 " then " 2973 " else " 2974 " hmeblk_next + phys-x@ false " 2975 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2976 " then " 2977 " else " 2978 " true " 2979 " then " 2980 " until r> drop " 2981 "; " 2982 2983 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2984 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2985 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2986 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2987 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2988 "; " 2989 2990 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2991 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2992 " TTE8K = if ( hmeblkp addr ) " 2993 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2994 " else ( hmeblkp addr ) " 2995 " drop 0 ( hmeblkp 0 ) " 2996 " then ( hmeblkp hme-index ) " 2997 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2998 " SFHME_TTE + ( ttep ) " 2999 "; " 3000 3001 ": unix-tte ( addr cnum -- false | tte-data true ) " 3002 " KCONTEXT = if ( addr ) " 3003 " KHATID ( addr khatid ) " 3004 " else ( addr ) " 3005 " drop false exit ( false ) " 3006 " then " 3007 " ( addr khatid ) " 3008 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 3009 " 2dup swap i HME_HASH_SHIFT " 3010 "( addr sfmmup sfmmup addr hmeshift ) " 3011 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 3012 " over i 4 pick " 3013 "( addr sfmmup hmebp sfmmup rehash addr ) " 3014 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 3015 " HME_HASH_TABLE_SEARCH " 3016 "( addr sfmmup { null | hmeblkp } ) " 3017 " ?dup if ( addr sfmmup hmeblkp ) " 3018 " nip swap HBLK_TO_TTEP ( ttep ) " 3019 " dup TTE_IS_VALID if ( valid-ttep ) " 3020 " PHYS-X@ true ( tte-data true ) " 3021 " else ( invalid-tte ) " 3022 " drop false ( false ) " 3023 " then ( false | tte-data true ) " 3024 " unloop exit ( false | tte-data true ) " 3025 " then ( addr sfmmup ) " 3026 " loop ( addr sfmmup ) " 3027 " 2drop false ( false ) " 3028 "; " 3029 ; 3030 3031 void 3032 create_va_to_tte(void) 3033 { 3034 char *bp; 3035 extern int khmehash_num, uhmehash_num; 3036 extern struct hmehash_bucket *khme_hash, *uhme_hash; 3037 3038 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 3039 3040 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 3041 3042 /* 3043 * Teach obp how to parse our sw ttes. 3044 */ 3045 (void) sprintf(bp, obp_tte_str, 3046 MMU_PAGESHIFT, 3047 TTE8K, 3048 sizeof (struct sf_hment), 3049 OFFSET(struct sf_hment, hme_tte), 3050 OFFSET(struct hme_blk, hblk_tag), 3051 OFFSET(struct hme_blk, hblk_nextpa), 3052 OFFSET(struct hme_blk, hblk_misc), 3053 OFFSET(struct hme_blk, hblk_hme), 3054 NHMENTS, 3055 HBLK_SZMASK, 3056 HBLK_RANGE_SHIFT, 3057 OFFSET(struct hmehash_bucket, hmeh_nextpa), 3058 sizeof (struct hmehash_bucket), 3059 HTAG_SFMMUPSZ, 3060 HTAG_BSPAGE_SHIFT, 3061 HTAG_REHASH_SHIFT, 3062 SFMMU_INVALID_SHMERID, 3063 mmu_hashcnt, 3064 (caddr_t)va_to_pa((caddr_t)uhme_hash), 3065 (caddr_t)va_to_pa((caddr_t)khme_hash), 3066 UHMEHASH_SZ, 3067 KHMEHASH_SZ, 3068 KCONTEXT, 3069 KHATID, 3070 ASI_MEM); 3071 prom_interpret(bp, 0, 0, 0, 0, 0); 3072 3073 kobj_free(bp, MMU_PAGESIZE); 3074 } 3075 3076 void 3077 install_va_to_tte(void) 3078 { 3079 /* 3080 * advise prom that he can use unix-tte 3081 */ 3082 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3083 } 3084 3085 /* 3086 * Here we add "device-type=console" for /os-io node, for currently 3087 * our kernel console output only supports displaying text and 3088 * performing cursor-positioning operations (through kernel framebuffer 3089 * driver) and it doesn't support other functionalities required for a 3090 * standard "display" device as specified in 1275 spec. The main missing 3091 * interface defined by the 1275 spec is "draw-logo". 3092 * also see the comments above prom_stdout_is_framebuffer(). 3093 */ 3094 static char *create_node = 3095 "\" /\" find-device " 3096 "new-device " 3097 "\" os-io\" device-name " 3098 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3099 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3100 " 2>r swap 2 2r> ['] $callback catch if " 3101 " 2drop 3drop 0 " 3102 " then " 3103 "; " 3104 ": read ( adr,len -- #read ) " 3105 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3106 " ( retN ... ret1 N ) " 3107 " ?dup if " 3108 " swap >r 1- 0 ?do drop loop r> " 3109 " else " 3110 " -2 " 3111 " then " 3112 "; " 3113 ": write ( adr,len -- #written ) " 3114 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3115 " ( retN ... ret1 N ) " 3116 " ?dup if " 3117 " swap >r 1- 0 ?do drop loop r> " 3118 " else " 3119 " 0 " 3120 " then " 3121 "; " 3122 ": poll-tty ( -- ) ; " 3123 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3124 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3125 ": cb-give/take ( $method -- ) " 3126 " 0 -rot ['] $callback catch ?dup if " 3127 " >r 2drop 2drop r> throw " 3128 " else " 3129 " 0 ?do drop loop " 3130 " then " 3131 "; " 3132 ": give ( -- ) \" exit-input\" cb-give/take ; " 3133 ": take ( -- ) \" enter-input\" cb-give/take ; " 3134 ": open ( -- ok? ) true ; " 3135 ": close ( -- ) ; " 3136 "finish-device " 3137 "device-end "; 3138 3139 /* 3140 * Create the OBP input/output node (FCode serial driver). 3141 * It is needed for both USB console keyboard and for 3142 * the kernel terminal emulator. It is too early to check for a 3143 * kernel console compatible framebuffer now, so we create this 3144 * so that we're ready if we need to enable kernel terminal emulation. 3145 * 3146 * When the USB software takes over the input device at the time 3147 * consconfig runs, OBP's stdin is redirected to this node. 3148 * Whenever the FORTH user interface is used after this switch, 3149 * the node will call back into the kernel for console input. 3150 * If a serial device such as ttya or a UART with a Type 5 keyboard 3151 * attached is used, OBP takes over the serial device when the system 3152 * goes to the debugger after the system is booted. This sharing 3153 * of the relatively simple serial device is difficult but possible. 3154 * Sharing the USB host controller is impossible due its complexity. 3155 * 3156 * Similarly to USB keyboard input redirection, after consconfig_dacf 3157 * configures a kernel console framebuffer as the standard output 3158 * device, OBP's stdout is switched to to vector through the 3159 * /os-io node into the kernel terminal emulator. 3160 */ 3161 static void 3162 startup_create_io_node(void) 3163 { 3164 prom_interpret(create_node, 0, 0, 0, 0, 0); 3165 } 3166 3167 3168 static void 3169 do_prom_version_check(void) 3170 { 3171 int i; 3172 pnode_t node; 3173 char buf[64]; 3174 static char drev[] = "Down-rev firmware detected%s\n" 3175 "\tPlease upgrade to the following minimum version:\n" 3176 "\t\t%s\n"; 3177 3178 i = prom_version_check(buf, sizeof (buf), &node); 3179 3180 if (i == PROM_VER64_OK) 3181 return; 3182 3183 if (i == PROM_VER64_UPGRADE) { 3184 cmn_err(CE_WARN, drev, "", buf); 3185 3186 #ifdef DEBUG 3187 prom_enter_mon(); /* Type 'go' to continue */ 3188 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3189 return; 3190 #else 3191 halt(0); 3192 #endif 3193 } 3194 3195 /* 3196 * The other possibility is that this is a server running 3197 * good firmware, but down-rev firmware was detected on at 3198 * least one other cpu board. We just complain if we see 3199 * that. 3200 */ 3201 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3202 } 3203 3204 static void 3205 kpm_init() 3206 { 3207 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 3208 kpm_pgsz = 1ull << kpm_pgshft; 3209 kpm_pgoff = kpm_pgsz - 1; 3210 kpmp2pshft = kpm_pgshft - PAGESHIFT; 3211 kpmpnpgs = 1 << kpmp2pshft; 3212 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 3213 } 3214 3215 void 3216 kpm_npages_setup(int memblocks) 3217 { 3218 /* 3219 * npages can be scattered in a maximum of 'memblocks' 3220 */ 3221 kpm_npages = ptokpmpr(npages) + memblocks; 3222 } 3223 3224 /* 3225 * Must be defined in platform dependent code. 3226 */ 3227 extern caddr_t modtext; 3228 extern size_t modtext_sz; 3229 extern caddr_t moddata; 3230 3231 #define HEAPTEXT_ARENA(addr) \ 3232 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3233 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3234 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3235 3236 #define HEAPTEXT_OVERSIZED(addr) \ 3237 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3238 3239 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3240 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3241 kmutex_t texthole_lock; 3242 3243 char kern_bootargs[OBP_MAXPATHLEN]; 3244 3245 void 3246 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3247 { 3248 uintptr_t addr, limit; 3249 3250 addr = HEAPTEXT_BASE; 3251 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3252 3253 /* 3254 * Before we initialize the text_arena, we want to punch holes in the 3255 * underlying heaptext_arena. This guarantees that for any text 3256 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3257 */ 3258 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3259 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3260 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3261 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3262 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3263 } 3264 3265 /* 3266 * Allocate one page at the oversize to break up the text region 3267 * from the oversized region. 3268 */ 3269 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3270 (void *)limit, (void *)(limit + PAGESIZE), 3271 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3272 3273 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3274 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3275 heaptext_arena, 0, VM_SLEEP); 3276 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3277 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3278 } 3279 3280 caddr_t 3281 kobj_text_alloc(vmem_t *arena, size_t size) 3282 { 3283 caddr_t rval, better; 3284 3285 /* 3286 * First, try a sleeping allocation. 3287 */ 3288 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3289 3290 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3291 return (rval); 3292 3293 /* 3294 * We didn't get the area that we wanted. We're going to try to do an 3295 * allocation with explicit constraints. 3296 */ 3297 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3298 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3299 VM_NOSLEEP | VM_BESTFIT); 3300 3301 if (better != NULL) { 3302 /* 3303 * That worked. Free our first attempt and return. 3304 */ 3305 vmem_free(arena, rval, size); 3306 return (better); 3307 } 3308 3309 /* 3310 * That didn't work; we'll have to return our first attempt. 3311 */ 3312 return (rval); 3313 } 3314 3315 caddr_t 3316 kobj_texthole_alloc(caddr_t addr, size_t size) 3317 { 3318 int arena = HEAPTEXT_ARENA(addr); 3319 char c[30]; 3320 uintptr_t base; 3321 3322 if (HEAPTEXT_OVERSIZED(addr)) { 3323 /* 3324 * If this is an oversized allocation, there is no text hole 3325 * available for it; return NULL. 3326 */ 3327 return (NULL); 3328 } 3329 3330 mutex_enter(&texthole_lock); 3331 3332 if (texthole_arena[arena] == NULL) { 3333 ASSERT(texthole_source[arena] == NULL); 3334 3335 if (arena == 0) { 3336 texthole_source[0] = vmem_create("module_text_holesrc", 3337 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3338 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3339 0, VM_SLEEP); 3340 } else { 3341 base = HEAPTEXT_BASE + 3342 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3343 3344 (void) snprintf(c, sizeof (c), 3345 "heaptext_holesrc_%d", arena); 3346 3347 texthole_source[arena] = vmem_create(c, (void *)base, 3348 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3349 0, VM_SLEEP); 3350 } 3351 3352 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3353 3354 texthole_arena[arena] = vmem_create(c, NULL, 0, 3355 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3356 texthole_source[arena], 0, VM_SLEEP); 3357 } 3358 3359 mutex_exit(&texthole_lock); 3360 3361 ASSERT(texthole_arena[arena] != NULL); 3362 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3363 return (vmem_alloc(texthole_arena[arena], size, 3364 VM_BESTFIT | VM_NOSLEEP)); 3365 } 3366 3367 void 3368 kobj_texthole_free(caddr_t addr, size_t size) 3369 { 3370 int arena = HEAPTEXT_ARENA(addr); 3371 3372 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3373 ASSERT(texthole_arena[arena] != NULL); 3374 vmem_free(texthole_arena[arena], addr, size); 3375 } 3376