1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern int cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 74 extern void mach_dump_buffer_init(void); 75 extern void mach_descrip_init(void); 76 extern void mach_descrip_startup_fini(void); 77 extern void mach_memscrub(void); 78 extern void mach_fpras(void); 79 extern void mach_cpu_halt_idle(void); 80 extern void mach_hw_copy_limit(void); 81 extern void load_mach_drivers(void); 82 extern void load_tod_module(void); 83 #pragma weak load_tod_module 84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata); 86 #pragma weak ndata_alloc_mmfsa 87 88 extern void cif_init(void); 89 #pragma weak cif_init 90 91 extern void parse_idprom(void); 92 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 93 extern void mem_config_init(void); 94 extern void memseg_remap_init(void); 95 96 extern void mach_kpm_init(void); 97 98 /* 99 * External Data: 100 */ 101 extern int vac_size; /* cache size in bytes */ 102 extern uint_t vac_mask; /* VAC alignment consistency mask */ 103 extern uint_t vac_colors; 104 105 /* 106 * Global Data Definitions: 107 */ 108 109 /* 110 * XXX - Don't port this to new architectures 111 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 112 * 'romp' has no use with a prom with an IEEE 1275 client interface. 113 * The driver doesn't use the value, but it depends on the symbol. 114 */ 115 void *romp; /* veritas driver won't load without romp 4154976 */ 116 /* 117 * Declare these as initialized data so we can patch them. 118 */ 119 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 120 pgcnt_t segkpsize = 121 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 122 uint_t segmap_percent = 12; /* Size of segmap segment */ 123 124 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 125 int vac_copyback = 1; 126 char *cache_mode = NULL; 127 int use_mix = 1; 128 int prom_debug = 0; 129 130 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 131 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 132 uint_t tba_taken_over = 0; 133 134 caddr_t s_text; /* start of kernel text segment */ 135 caddr_t e_text; /* end of kernel text segment */ 136 caddr_t s_data; /* start of kernel data segment */ 137 caddr_t e_data; /* end of kernel data segment */ 138 139 caddr_t modtext; /* beginning of module text */ 140 size_t modtext_sz; /* size of module text */ 141 caddr_t moddata; /* beginning of module data reserve */ 142 caddr_t e_moddata; /* end of module data reserve */ 143 144 /* 145 * End of first block of contiguous kernel in 32-bit virtual address space 146 */ 147 caddr_t econtig32; /* end of first blk of contiguous kernel */ 148 149 caddr_t ncbase; /* beginning of non-cached segment */ 150 caddr_t ncend; /* end of non-cached segment */ 151 caddr_t sdata; /* beginning of data segment */ 152 153 caddr_t extra_etva; /* beginning of unused nucleus text */ 154 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 155 156 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 157 caddr_t nalloc_base; /* beginning of nucleus allocation */ 158 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 159 caddr_t valloc_base; /* beginning of kvalloc segment */ 160 161 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 162 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 163 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 164 int kmem64_alignsize; /* page size for mem segment in 64-bit space */ 165 int kmem64_szc; /* page size code */ 166 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 167 168 uintptr_t shm_alignment; /* VAC address consistency modulus */ 169 struct memlist *phys_install; /* Total installed physical memory */ 170 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 171 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 172 struct memlist ndata; /* memlist of nucleus allocatable memory */ 173 int memexp_flag; /* memory expansion card flag */ 174 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 175 pgcnt_t obp_pages; /* Physical pages used by OBP */ 176 177 /* 178 * VM data structures 179 */ 180 long page_hashsz; /* Size of page hash table (power of two) */ 181 struct page *pp_base; /* Base of system page struct array */ 182 size_t pp_sz; /* Size in bytes of page struct array */ 183 struct page **page_hash; /* Page hash table */ 184 struct seg ktextseg; /* Segment used for kernel executable image */ 185 struct seg kvalloc; /* Segment used for "valloc" mapping */ 186 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 187 struct seg ktexthole; /* Segment used for nucleus text hole */ 188 struct seg kmapseg; /* Segment used for generic kernel mappings */ 189 struct seg kpmseg; /* Segment used for physical mapping */ 190 struct seg kdebugseg; /* Segment used for the kernel debugger */ 191 192 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 193 size_t kpm_pp_sz; /* Size of system kpm_page array */ 194 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 195 196 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 197 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 198 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 199 200 int segzio_fromheap = 0; /* zio allocations occur from heap */ 201 caddr_t segzio_base; /* Base address of segzio */ 202 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 203 204 /* 205 * debugger pages (if allocated) 206 */ 207 struct vnode kdebugvp; 208 209 /* 210 * VA range available to the debugger 211 */ 212 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 213 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 214 215 /* 216 * Segment for relocated kernel structures in 64-bit large RAM kernels 217 */ 218 struct seg kmem64; 219 220 struct memseg *memseg_base; 221 size_t memseg_sz; /* Used to translate a va to page */ 222 struct vnode unused_pages_vp; 223 224 /* 225 * VM data structures allocated early during boot. 226 */ 227 size_t pagehash_sz; 228 uint64_t memlist_sz; 229 230 char tbr_wr_addr_inited = 0; 231 232 233 /* 234 * Static Routines: 235 */ 236 static void memlist_add(uint64_t, uint64_t, struct memlist **, 237 struct memlist **); 238 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 239 pgcnt_t); 240 static void kvm_init(void); 241 242 static void startup_init(void); 243 static void startup_memlist(void); 244 static void startup_modules(void); 245 static void startup_bop_gone(void); 246 static void startup_vm(void); 247 static void startup_end(void); 248 static void setup_cage_params(void); 249 static void startup_create_io_node(void); 250 251 static pgcnt_t npages; 252 static struct memlist *memlist; 253 void *memlist_end; 254 255 static pgcnt_t bop_alloc_pages; 256 static caddr_t hblk_base; 257 uint_t hblk_alloc_dynamic = 0; 258 uint_t hblk1_min = H1MIN; 259 260 261 /* 262 * Hooks for unsupported platforms and down-rev firmware 263 */ 264 int iam_positron(void); 265 #pragma weak iam_positron 266 static void do_prom_version_check(void); 267 static void kpm_init(void); 268 static void kpm_npages_setup(int); 269 static void kpm_memseg_init(void); 270 271 /* 272 * After receiving a thermal interrupt, this is the number of seconds 273 * to delay before shutting off the system, assuming 274 * shutdown fails. Use /etc/system to change the delay if this isn't 275 * large enough. 276 */ 277 int thermal_powerdown_delay = 1200; 278 279 /* 280 * Used to hold off page relocations into the cage until OBP has completed 281 * its boot-time handoff of its resources to the kernel. 282 */ 283 int page_relocate_ready = 0; 284 285 /* 286 * Enable some debugging messages concerning memory usage... 287 */ 288 #ifdef DEBUGGING_MEM 289 static int debugging_mem; 290 static void 291 printmemlist(char *title, struct memlist *list) 292 { 293 if (!debugging_mem) 294 return; 295 296 printf("%s\n", title); 297 298 while (list) { 299 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 300 (uint32_t)(list->address >> 32), (uint32_t)list->address, 301 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 302 list = list->next; 303 } 304 } 305 306 void 307 printmemseg(struct memseg *memseg) 308 { 309 if (!debugging_mem) 310 return; 311 312 printf("memseg\n"); 313 314 while (memseg) { 315 prom_printf("\tpage = 0x%p, epage = 0x%p, " 316 "pfn = 0x%x, epfn = 0x%x\n", 317 memseg->pages, memseg->epages, 318 memseg->pages_base, memseg->pages_end); 319 memseg = memseg->next; 320 } 321 } 322 323 #define debug_pause(str) halt((str)) 324 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 325 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 326 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 327 #define MPRINTF3(str, a, b, c) \ 328 if (debugging_mem) prom_printf((str), (a), (b), (c)) 329 #else /* DEBUGGING_MEM */ 330 #define MPRINTF(str) 331 #define MPRINTF1(str, a) 332 #define MPRINTF2(str, a, b) 333 #define MPRINTF3(str, a, b, c) 334 #endif /* DEBUGGING_MEM */ 335 336 /* Simple message to indicate that the bootops pointer has been zeroed */ 337 #ifdef DEBUG 338 static int bootops_gone_on = 0; 339 #define BOOTOPS_GONE() \ 340 if (bootops_gone_on) \ 341 prom_printf("The bootops vec is zeroed now!\n"); 342 #else 343 #define BOOTOPS_GONE() 344 #endif /* DEBUG */ 345 346 /* 347 * Monitor pages may not be where this says they are. 348 * and the debugger may not be there either. 349 * 350 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 351 * 352 * Physical memory layout 353 * (not necessarily contiguous) 354 * (THIS IS SOMEWHAT WRONG) 355 * /-----------------------\ 356 * | monitor pages | 357 * availmem -|-----------------------| 358 * | | 359 * | page pool | 360 * | | 361 * |-----------------------| 362 * | configured tables | 363 * | buffers | 364 * firstaddr -|-----------------------| 365 * | hat data structures | 366 * |-----------------------| 367 * | kernel data, bss | 368 * |-----------------------| 369 * | interrupt stack | 370 * |-----------------------| 371 * | kernel text (RO) | 372 * |-----------------------| 373 * | trap table (4k) | 374 * |-----------------------| 375 * page 1 | panicbuf | 376 * |-----------------------| 377 * page 0 | reclaimed | 378 * |_______________________| 379 * 380 * 381 * 382 * Kernel's Virtual Memory Layout. 383 * /-----------------------\ 384 * 0xFFFFFFFF.FFFFFFFF -| |- 385 * | OBP's virtual page | 386 * | tables | 387 * 0xFFFFFFFC.00000000 -|-----------------------|- 388 * : : 389 * : : 390 * -|-----------------------|- 391 * | segzio | (base and size vary) 392 * 0xFFFFFE00.00000000 -|-----------------------|- 393 * | | Ultrasparc I/II support 394 * | segkpm segment | up to 2TB of physical 395 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 396 * | | 397 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 398 * : : 399 * : : 400 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 401 * | | ^ 402 * | UltraSPARC I/II call | | 403 * | bug requires an extra | | 404 * | 4 GB of space between | | 405 * | hole and used RAM | | 406 * | | | 407 * 0xFFFFF800.00000000 -|-----------------------|- | 408 * | | | 409 * | Virtual Address Hole | UltraSPARC 410 * | on UltraSPARC I/II | I/II * ONLY * 411 * | | | 412 * 0x00000800.00000000 -|-----------------------|- | 413 * | | | 414 * | UltraSPARC I/II call | | 415 * | bug requires an extra | | 416 * | 4 GB of space between | | 417 * | hole and used RAM | | 418 * | | v 419 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 420 * : : ^ 421 * : : | 422 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 423 * | overmapped area | alignend_end | 424 * | (kmem64_alignsize | | 425 * | boundary) | | 426 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 427 * | | | 428 * | 64-bit kernel ONLY | | 429 * | | | 430 * | kmem64 segment | | 431 * | | | 432 * | (Relocated extra HME | Approximately 433 * | block allocations, | 1 TB of virtual 434 * | memnode freelists, | address space 435 * | HME hash buckets, | | 436 * | mml_table, kpmp_table,| | 437 * | page_t array and | | 438 * | hashblock pool to | | 439 * | avoid hard-coded | | 440 * | 32-bit vaddr | | 441 * | limitations) | | 442 * | | v 443 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 444 * | | 445 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 446 * | | 447 * 0x00000300.00000000 -|-----------------------|- SYSBASE 448 * : : 449 * : : 450 * -|-----------------------|- 451 * | | 452 * | segmap segment | SEGMAPSIZE (1/8th physmem, 453 * | | 256G MAX) 454 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 455 * : : 456 * : : 457 * -|-----------------------|- 458 * | | 459 * | segkp | SEGKPSIZE (2GB) 460 * | | 461 * | | 462 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 463 * | | 464 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 465 * | | (SEGKPBASE - 0x400000) 466 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 467 * | | (MEMSCRUBBASE - NCARGS) 468 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 469 * | | (ARGSBASE - PPMAPSIZE) 470 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 471 * | | 472 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 473 * | | 474 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 475 * : : 476 * : : 477 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 478 * | | 479 * | OBP | 480 * | | 481 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 482 * | kmdb | 483 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 484 * : : 485 * : : 486 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 487 * | | 488 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 489 * | | ~64MB) 490 * 0x00000000.78002000 -|-----------------------| 491 * | panicbuf | 492 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 493 * : : 494 * : : 495 * | | 496 * |-----------------------|- econtig32 497 * | vm structures | 498 * 0x00000000.01C00000 |-----------------------|- nalloc_end 499 * | TSBs | 500 * |-----------------------|- end/nalloc_base 501 * | kernel data & bss | 502 * 0x00000000.01800000 -|-----------------------| 503 * : nucleus text hole : 504 * 0x00000000.01400000 -|-----------------------| 505 * : : 506 * |-----------------------| 507 * | module text | 508 * |-----------------------|- e_text/modtext 509 * | kernel text | 510 * |-----------------------| 511 * | trap table (48k) | 512 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 513 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 514 * |-----------------------| 515 * | | 516 * | invalid | 517 * | | 518 * 0x00000000.00000000 _|_______________________| 519 * 520 * 521 * 522 * 32-bit User Virtual Memory Layout. 523 * /-----------------------\ 524 * | | 525 * | invalid | 526 * | | 527 * 0xFFC00000 -|-----------------------|- USERLIMIT 528 * | user stack | 529 * : : 530 * : : 531 * : : 532 * | user data | 533 * -|-----------------------|- 534 * | user text | 535 * 0x00002000 -|-----------------------|- 536 * | invalid | 537 * 0x00000000 _|_______________________| 538 * 539 * 540 * 541 * 64-bit User Virtual Memory Layout. 542 * /-----------------------\ 543 * | | 544 * | invalid | 545 * | | 546 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 547 * | user stack | 548 * : : 549 * : : 550 * : : 551 * | user data | 552 * -|-----------------------|- 553 * | user text | 554 * 0x00000000.00100000 -|-----------------------|- 555 * | invalid | 556 * 0x00000000.00000000 _|_______________________| 557 */ 558 559 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 560 extern uint64_t ecache_flush_address(void); 561 562 #pragma weak load_platform_modules 563 #pragma weak plat_startup_memlist 564 #pragma weak ecache_init_scrub_flush_area 565 #pragma weak ecache_flush_address 566 567 568 /* 569 * By default the DR Cage is enabled for maximum OS 570 * MPSS performance. Users needing to disable the cage mechanism 571 * can set this variable to zero via /etc/system. 572 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 573 * will result in loss of DR functionality. 574 * Platforms wishing to disable kernel Cage by default 575 * should do so in their set_platform_defaults() routine. 576 */ 577 int kernel_cage_enable = 1; 578 579 static void 580 setup_cage_params(void) 581 { 582 void (*func)(void); 583 584 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 585 if (func != NULL) { 586 (*func)(); 587 return; 588 } 589 590 if (kernel_cage_enable == 0) { 591 return; 592 } 593 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 594 595 if (kcage_on) { 596 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 597 } else { 598 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 599 } 600 601 } 602 603 /* 604 * Machine-dependent startup code 605 */ 606 void 607 startup(void) 608 { 609 startup_init(); 610 if (&startup_platform) 611 startup_platform(); 612 startup_memlist(); 613 startup_modules(); 614 setup_cage_params(); 615 startup_bop_gone(); 616 startup_vm(); 617 startup_end(); 618 } 619 620 struct regs sync_reg_buf; 621 uint64_t sync_tt; 622 623 void 624 sync_handler(void) 625 { 626 struct trap_info ti; 627 int i; 628 629 /* 630 * Prevent trying to talk to the other CPUs since they are 631 * sitting in the prom and won't reply. 632 */ 633 for (i = 0; i < NCPU; i++) { 634 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 635 cpu[i]->cpu_flags &= ~CPU_READY; 636 cpu[i]->cpu_flags |= CPU_QUIESCED; 637 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 638 } 639 } 640 641 /* 642 * We've managed to get here without going through the 643 * normal panic code path. Try and save some useful 644 * information. 645 */ 646 if (!panicstr && (curthread->t_panic_trap == NULL)) { 647 ti.trap_type = sync_tt; 648 ti.trap_regs = &sync_reg_buf; 649 ti.trap_addr = NULL; 650 ti.trap_mmu_fsr = 0x0; 651 652 curthread->t_panic_trap = &ti; 653 } 654 655 /* 656 * If we're re-entering the panic path, update the signature 657 * block so that the SC knows we're in the second part of panic. 658 */ 659 if (panicstr) 660 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 661 662 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 663 panic("sync initiated"); 664 } 665 666 667 static void 668 startup_init(void) 669 { 670 /* 671 * We want to save the registers while we're still in OBP 672 * so that we know they haven't been fiddled with since. 673 * (In principle, OBP can't change them just because it 674 * makes a callback, but we'd rather not depend on that 675 * behavior.) 676 */ 677 char sync_str[] = 678 "warning @ warning off : sync " 679 "%%tl-c %%tstate h# %p x! " 680 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 681 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 682 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 683 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 684 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 685 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 686 "%%y h# %p l! %%tl-c %%tt h# %p x! " 687 "sync ; warning !"; 688 689 /* 690 * 20 == num of %p substrings 691 * 16 == max num of chars %p will expand to. 692 */ 693 char bp[sizeof (sync_str) + 16 * 20]; 694 695 (void) check_boot_version(BOP_GETVERSION(bootops)); 696 697 /* 698 * Initialize ptl1 stack for the 1st CPU. 699 */ 700 ptl1_init_cpu(&cpu0); 701 702 /* 703 * Initialize the address map for cache consistent mappings 704 * to random pages; must be done after vac_size is set. 705 */ 706 ppmapinit(); 707 708 /* 709 * Initialize the PROM callback handler. 710 */ 711 init_vx_handler(); 712 713 /* 714 * have prom call sync_callback() to handle the sync and 715 * save some useful information which will be stored in the 716 * core file later. 717 */ 718 (void) sprintf((char *)bp, sync_str, 719 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 720 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 721 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 722 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 723 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 724 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 725 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 726 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 727 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 728 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 729 prom_interpret(bp, 0, 0, 0, 0, 0); 730 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 731 } 732 733 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 734 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 735 736 #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 737 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 738 (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 739 740 #if !defined(C_OBP) 741 /* 742 * Install a temporary tte handler in OBP for kmem64 area. 743 * 744 * We map kmem64 area with large pages before the trap table is taken 745 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 746 * the same area. Duplicate tlb entries with different page sizes 747 * cause unpredicatble behavior. To avoid this, we don't create 748 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 749 * OBP). Instead, we manage translations with a temporary va>tte-data 750 * handler (kmem64-tte). This handler is replaced by unix-tte when 751 * the trap table is taken over. 752 * 753 * The temporary handler knows the physical address of the kmem64 754 * area. It uses the prom's pgmap@ Forth word for other addresses. 755 * 756 * We have to use BOP_ALLOC() method for C-OBP platforms because 757 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 758 * sun4u platforms. On sun4u we flush tlb after trap table is taken 759 * over if we use large pages for kernel heap and kmem64. Since sun4u 760 * prom (unlike sun4v) calls va>tte-data first for client address 761 * translation prom's ttes for kmem64 can't get into TLB even if we 762 * later switch to prom's trap table again. C-OBP uses 4M pages for 763 * client mappings when possible so on all platforms we get the 764 * benefit from large mappings for kmem64 area immediately during 765 * boot. 766 * 767 * pseudo code: 768 * if (context != 0) { 769 * return false 770 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 771 * tte = tte_template + 772 * (((miss_va & pagemask) - kmem64_base)); 773 * return tte, true 774 * } else { 775 * return pgmap@ result 776 * } 777 */ 778 char kmem64_obp_str[] = 779 "h# %lx constant kmem64_base " 780 "h# %lx constant kmem64_end " 781 "h# %lx constant kmem64_pagemask " 782 "h# %lx constant kmem64_template " 783 784 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 785 " if ( addr ) " 786 " drop false exit then ( false ) " 787 " dup kmem64_base kmem64_end within if ( addr ) " 788 " kmem64_pagemask and ( addr' ) " 789 " kmem64_base - ( addr' ) " 790 " kmem64_template + ( tte ) " 791 " true ( tte true ) " 792 " else ( addr ) " 793 " pgmap@ ( tte ) " 794 " dup 0< if true else drop false then ( tte true | false ) " 795 " then ( tte true | false ) " 796 "; " 797 798 "' kmem64-tte is va>tte-data " 799 ; 800 801 void 802 install_kmem64_tte() 803 { 804 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 805 tte_t tte; 806 807 PRM_DEBUG(kmem64_pabase); 808 PRM_DEBUG(kmem64_szc); 809 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 810 PROC_DATA | HAT_NOSYNC, kmem64_szc); 811 PRM_DEBUG(tte.ll); 812 (void) sprintf(b, kmem64_obp_str, 813 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 814 ASSERT(strlen(b) < sizeof (b)); 815 prom_interpret(b, 0, 0, 0, 0, 0); 816 } 817 #endif /* !C_OBP */ 818 819 /* 820 * As OBP takes up some RAM when the system boots, pages will already be "lost" 821 * to the system and reflected in npages by the time we see it. 822 * 823 * We only want to allocate kernel structures in the 64-bit virtual address 824 * space on systems with enough RAM to make the overhead of keeping track of 825 * an extra kernel memory segment worthwhile. 826 * 827 * Since OBP has already performed its memory allocations by this point, if we 828 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 829 * memory in the 64-bit virtual address space; otherwise keep allocations 830 * contiguous with we've mapped so far in the 32-bit virtual address space. 831 */ 832 #define MINMOVE_RAM_MB ((size_t)1900) 833 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 834 835 pgcnt_t tune_npages = (pgcnt_t) 836 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 837 838 #pragma weak page_set_colorequiv_arr_cpu 839 extern void page_set_colorequiv_arr_cpu(void); 840 841 static void 842 startup_memlist(void) 843 { 844 size_t alloc_sz; 845 size_t ctrs_sz; 846 caddr_t alloc_base; 847 caddr_t ctrs_base, ctrs_end; 848 caddr_t memspace; 849 caddr_t va; 850 int memblocks = 0; 851 struct memlist *cur; 852 size_t syslimit = (size_t)SYSLIMIT; 853 size_t sysbase = (size_t)SYSBASE; 854 int alloc_alignsize = ecache_alignsize; 855 int i; 856 extern void page_coloring_init(void); 857 extern void page_set_colorequiv_arr(void); 858 859 /* 860 * Initialize enough of the system to allow kmem_alloc to work by 861 * calling boot to allocate its memory until the time that 862 * kvm_init is completed. The page structs are allocated after 863 * rounding up end to the nearest page boundary; the memsegs are 864 * initialized and the space they use comes from the kernel heap. 865 * With appropriate initialization, they can be reallocated later 866 * to a size appropriate for the machine's configuration. 867 * 868 * At this point, memory is allocated for things that will never 869 * need to be freed, this used to be "valloced". This allows a 870 * savings as the pages don't need page structures to describe 871 * them because them will not be managed by the vm system. 872 */ 873 874 /* 875 * We're loaded by boot with the following configuration (as 876 * specified in the sun4u/conf/Mapfile): 877 * 878 * text: 4 MB chunk aligned on a 4MB boundary 879 * data & bss: 4 MB chunk aligned on a 4MB boundary 880 * 881 * These two chunks will eventually be mapped by 2 locked 4MB 882 * ttes and will represent the nucleus of the kernel. This gives 883 * us some free space that is already allocated, some or all of 884 * which is made available to kernel module text. 885 * 886 * The free space in the data-bss chunk is used for nucleus 887 * allocatable data structures and we reserve it using the 888 * nalloc_base and nalloc_end variables. This space is currently 889 * being used for hat data structures required for tlb miss 890 * handling operations. We align nalloc_base to a l2 cache 891 * linesize because this is the line size the hardware uses to 892 * maintain cache coherency. 893 * 256K is carved out for module data. 894 */ 895 896 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 897 moddata = nalloc_base; 898 e_moddata = nalloc_base + MODDATA; 899 nalloc_base = e_moddata; 900 901 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 902 valloc_base = nalloc_base; 903 904 /* 905 * Calculate the start of the data segment. 906 */ 907 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 908 909 PRM_DEBUG(moddata); 910 PRM_DEBUG(nalloc_base); 911 PRM_DEBUG(nalloc_end); 912 PRM_DEBUG(sdata); 913 914 /* 915 * Remember any slop after e_text so we can give it to the modules. 916 */ 917 PRM_DEBUG(e_text); 918 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 919 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 920 prom_panic("nucleus text overflow"); 921 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 922 modtext; 923 PRM_DEBUG(modtext); 924 PRM_DEBUG(modtext_sz); 925 926 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 927 &boot_physavail, &boot_physavail_len, 928 &boot_virtavail, &boot_virtavail_len); 929 /* 930 * Remember what the physically available highest page is 931 * so that dumpsys works properly, and find out how much 932 * memory is installed. 933 */ 934 installed_top_size_memlist_array(boot_physinstalled, 935 boot_physinstalled_len, &physmax, &physinstalled); 936 PRM_DEBUG(physinstalled); 937 PRM_DEBUG(physmax); 938 939 /* Fill out memory nodes config structure */ 940 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 941 942 /* 943 * Get the list of physically available memory to size 944 * the number of page structures needed. 945 */ 946 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 947 /* 948 * This first snap shot of npages can represent the pages used 949 * by OBP's text and data approximately. This is used in the 950 * the calculation of the kernel size 951 */ 952 obp_pages = physinstalled - npages; 953 954 955 /* 956 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 957 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 958 * 2MB) and any excess pages are put on physavail. The assumption is 959 * that small-memory systems will need more pages more than they'll 960 * need efficiently-mapped module texts. 961 */ 962 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 963 modtext_sz > MODTEXT_SM_CAP) { 964 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 965 modtext_sz = MODTEXT_SM_CAP; 966 extra_etva = modtext + modtext_sz; 967 } 968 969 PRM_DEBUG(extra_etpg); 970 PRM_DEBUG(modtext_sz); 971 PRM_DEBUG(extra_etva); 972 973 /* 974 * Account for any pages after e_text and e_data. 975 */ 976 npages += extra_etpg; 977 npages += mmu_btopr(nalloc_end - nalloc_base); 978 PRM_DEBUG(npages); 979 980 /* 981 * npages is the maximum of available physical memory possible. 982 * (ie. it will never be more than this) 983 */ 984 985 /* 986 * initialize the nucleus memory allocator. 987 */ 988 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 989 990 /* 991 * Allocate mmu fault status area from the nucleus data area. 992 */ 993 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 994 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 995 996 /* 997 * Allocate kernel TSBs from the nucleus data area. 998 */ 999 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1000 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1001 1002 /* 1003 * Allocate dmv dispatch table from the nucleus data area. 1004 */ 1005 if (ndata_alloc_dmv(&ndata) != 0) 1006 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1007 1008 1009 page_coloring_init(); 1010 1011 /* 1012 * Allocate page_freelists bin headers for memnode 0 from the 1013 * nucleus data area. 1014 */ 1015 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 1016 cmn_err(CE_PANIC, 1017 "no more nucleus memory after page free lists alloc"); 1018 1019 if (kpm_enable) { 1020 kpm_init(); 1021 /* 1022 * kpm page space -- Update kpm_npages and make the 1023 * same assumption about fragmenting as it is done 1024 * for memseg_sz. 1025 */ 1026 kpm_npages_setup(memblocks + 4); 1027 } 1028 1029 /* 1030 * Allocate hat related structs from the nucleus data area. 1031 */ 1032 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 1033 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1034 1035 /* 1036 * We want to do the BOP_ALLOCs before the real allocation of page 1037 * structs in order to not have to allocate page structs for this 1038 * memory. We need to calculate a virtual address because we want 1039 * the page structs to come before other allocations in virtual address 1040 * space. This is so some (if not all) of page structs can actually 1041 * live in the nucleus. 1042 */ 1043 1044 /* 1045 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1046 * 1047 * There are comments all over the SFMMU code warning of dire 1048 * consequences if the TSBs are moved out of 32-bit space. This 1049 * is largely because the asm code uses "sethi %hi(addr)"-type 1050 * instructions which will not provide the expected result if the 1051 * address is a 64-bit one. 1052 * 1053 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1054 */ 1055 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1056 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1057 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1058 PRM_DEBUG(alloc_base); 1059 1060 /* 1061 * Allocate IOMMU TSB array. We do this here so that the physical 1062 * memory gets deducted from the PROM's physical memory list. 1063 */ 1064 alloc_base = iommu_tsb_init(alloc_base); 1065 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1066 ecache_alignsize); 1067 PRM_DEBUG(alloc_base); 1068 1069 /* 1070 * Platforms like Starcat and OPL need special structures assigned in 1071 * 32-bit virtual address space because their probing routines execute 1072 * FCode, and FCode can't handle 64-bit virtual addresses... 1073 */ 1074 if (&plat_startup_memlist) { 1075 alloc_base = plat_startup_memlist(alloc_base); 1076 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1077 ecache_alignsize); 1078 PRM_DEBUG(alloc_base); 1079 } 1080 1081 /* 1082 * Save off where the contiguous allocations to date have ended 1083 * in econtig32. 1084 */ 1085 econtig32 = alloc_base; 1086 PRM_DEBUG(econtig32); 1087 1088 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1089 cmn_err(CE_PANIC, "econtig32 too big"); 1090 1091 /* 1092 * To avoid memory allocation collisions in the 32-bit virtual address 1093 * space, make allocations from this point forward in 64-bit virtual 1094 * address space starting at syslimit and working up. 1095 * 1096 * All this is needed because on large memory systems, the default 1097 * Solaris allocations will collide with SYSBASE32, which is hard 1098 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1099 * kernels, move the allocations to a location in the 64-bit virtual 1100 * address space space, allowing those structures to grow without 1101 * worry. 1102 * 1103 * On current CPUs we'll run out of physical memory address bits before 1104 * we need to worry about the allocations running into anything else in 1105 * VM or the virtual address holes on US-I and II, as there's currently 1106 * about 1 TB of addressable space before the US-I/II VA hole. 1107 */ 1108 kmem64_base = (caddr_t)syslimit; 1109 PRM_DEBUG(kmem64_base); 1110 1111 /* 1112 * Allocate addresses, but not physical memory. None of these locations 1113 * can be touched until physical memory is allocated below. 1114 */ 1115 alloc_base = kmem64_base; 1116 1117 /* 1118 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1119 * them here. 1120 */ 1121 if (khme_hash == NULL || uhme_hash == NULL) { 1122 /* 1123 * alloc_hme_buckets() will align alloc_base properly before 1124 * assigning the hash buckets, so we don't need to do it 1125 * before the call... 1126 */ 1127 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1128 1129 PRM_DEBUG(alloc_base); 1130 PRM_DEBUG(khme_hash); 1131 PRM_DEBUG(uhme_hash); 1132 } 1133 1134 /* 1135 * Allow for an early allocation of physically contiguous memory. 1136 */ 1137 alloc_base = contig_mem_prealloc(alloc_base, npages); 1138 1139 /* 1140 * Allocate the remaining page freelists. NUMA systems can 1141 * have lots of page freelists, one per node, which quickly 1142 * outgrow the amount of nucleus memory available. 1143 */ 1144 if (max_mem_nodes > 1) { 1145 int mnode; 1146 1147 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1148 alloc_base = alloc_page_freelists(mnode, alloc_base, 1149 ecache_alignsize); 1150 } 1151 PRM_DEBUG(alloc_base); 1152 } 1153 1154 if (!mml_table) { 1155 size_t mmltable_sz; 1156 1157 /* 1158 * We need to allocate the mml_table here because there 1159 * was not enough space within the nucleus. 1160 */ 1161 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1162 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1163 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1164 alloc_alignsize); 1165 mml_table = (kmutex_t *)alloc_base; 1166 alloc_base += alloc_sz; 1167 PRM_DEBUG(mml_table); 1168 PRM_DEBUG(alloc_base); 1169 } 1170 1171 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1172 size_t kpmptable_sz; 1173 caddr_t table; 1174 1175 /* 1176 * We need to allocate either kpmp_table or kpmp_stable here 1177 * because there was not enough space within the nucleus. 1178 */ 1179 kpmptable_sz = (kpm_smallpages == 0) ? 1180 sizeof (kpm_hlk_t) * kpmp_table_sz : 1181 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1182 1183 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1184 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1185 alloc_alignsize); 1186 1187 table = alloc_base; 1188 1189 if (kpm_smallpages == 0) { 1190 kpmp_table = (kpm_hlk_t *)table; 1191 PRM_DEBUG(kpmp_table); 1192 } else { 1193 kpmp_stable = (kpm_shlk_t *)table; 1194 PRM_DEBUG(kpmp_stable); 1195 } 1196 1197 alloc_base += alloc_sz; 1198 PRM_DEBUG(alloc_base); 1199 } 1200 1201 if (&ecache_init_scrub_flush_area) { 1202 /* 1203 * Pass alloc_base directly, as the routine itself is 1204 * responsible for any special alignment requirements... 1205 */ 1206 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1207 PRM_DEBUG(alloc_base); 1208 } 1209 1210 /* 1211 * Take the most current snapshot we can by calling mem-update. 1212 */ 1213 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1214 &boot_physavail, &boot_physavail_len, 1215 &boot_virtavail, &boot_virtavail_len); 1216 1217 /* 1218 * Reset npages and memblocks based on boot_physavail list. 1219 */ 1220 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1221 PRM_DEBUG(npages); 1222 1223 /* 1224 * Account for extra memory after e_text. 1225 */ 1226 npages += extra_etpg; 1227 1228 /* 1229 * Calculate the largest free memory chunk in the nucleus data area. 1230 * We need to figure out if page structs can fit in there or not. 1231 * We also make sure enough page structs get created for any physical 1232 * memory we might be returning to the system. 1233 */ 1234 ndata_remain_sz = ndata_maxsize(&ndata); 1235 PRM_DEBUG(ndata_remain_sz); 1236 1237 pp_sz = sizeof (struct page) * npages; 1238 1239 /* 1240 * Here's a nice bit of code based on somewhat recursive logic: 1241 * 1242 * If the page array would fit within the nucleus, we want to 1243 * add npages to cover any extra memory we may be returning back 1244 * to the system. 1245 * 1246 * HOWEVER, the page array is sized by calculating the size of 1247 * (struct page * npages), as are the pagehash table, ctrs and 1248 * memseg_list, so the very act of performing the calculation below may 1249 * in fact make the array large enough that it no longer fits in the 1250 * nucleus, meaning there would now be a much larger area of the 1251 * nucleus free that should really be added to npages, which would 1252 * make the page array that much larger, and so on. 1253 * 1254 * This also ignores the memory possibly used in the nucleus for the 1255 * the page hash, ctrs and memseg list and the fact that whether they 1256 * fit there or not varies with the npages calculation below, but we 1257 * don't even factor them into the equation at this point; perhaps we 1258 * should or perhaps we should just take the approach that the few 1259 * extra pages we could add via this calculation REALLY aren't worth 1260 * the hassle... 1261 */ 1262 if (ndata_remain_sz > pp_sz) { 1263 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1264 1265 npages += mmu_btop(spare); 1266 1267 pp_sz = npages * sizeof (struct page); 1268 1269 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1270 } 1271 1272 /* 1273 * If physmem is patched to be non-zero, use it instead of 1274 * the monitor value unless physmem is larger than the total 1275 * amount of memory on hand. 1276 */ 1277 if (physmem == 0 || physmem > npages) 1278 physmem = npages; 1279 1280 /* 1281 * If pp_base is NULL that means the routines above have determined 1282 * the page array will not fit in the nucleus; we'll have to 1283 * BOP_ALLOC() ourselves some space for them. 1284 */ 1285 if (pp_base == NULL) { 1286 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1287 alloc_alignsize); 1288 alloc_sz = roundup(pp_sz, alloc_alignsize); 1289 1290 pp_base = (struct page *)alloc_base; 1291 1292 alloc_base += alloc_sz; 1293 } 1294 1295 /* 1296 * The page structure hash table size is a power of 2 1297 * such that the average hash chain length is PAGE_HASHAVELEN. 1298 */ 1299 page_hashsz = npages / PAGE_HASHAVELEN; 1300 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1301 pagehash_sz = sizeof (struct page *) * page_hashsz; 1302 1303 /* 1304 * We want to TRY to fit the page structure hash table, 1305 * the page size free list counters, the memseg list and 1306 * and the kpm page space in the nucleus if possible. 1307 * 1308 * alloc_sz counts how much memory needs to be allocated by 1309 * BOP_ALLOC(). 1310 */ 1311 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1312 1313 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1314 1315 /* 1316 * Size up per page size free list counters. 1317 */ 1318 ctrs_sz = page_ctrs_sz(); 1319 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1320 1321 if (ctrs_base == NULL) 1322 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1323 1324 /* 1325 * The memseg list is for the chunks of physical memory that 1326 * will be managed by the vm system. The number calculated is 1327 * a guess as boot may fragment it more when memory allocations 1328 * are made before kphysm_init(). Currently, there are two 1329 * allocations before then, so we assume each causes fragmen- 1330 * tation, and add a couple more for good measure. 1331 */ 1332 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1333 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1334 1335 if (memseg_base == NULL) 1336 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1337 1338 1339 if (kpm_enable) { 1340 /* 1341 * kpm page space -- Update kpm_npages and make the 1342 * same assumption about fragmenting as it is done 1343 * for memseg_sz above. 1344 */ 1345 kpm_npages_setup(memblocks + 4); 1346 kpm_pp_sz = (kpm_smallpages == 0) ? 1347 kpm_npages * sizeof (kpm_page_t): 1348 kpm_npages * sizeof (kpm_spage_t); 1349 1350 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1351 ecache_alignsize); 1352 1353 if (kpm_pp_base == NULL) 1354 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1355 kpm_pp_sz; 1356 } 1357 1358 if (alloc_sz > 0) { 1359 uintptr_t bop_base; 1360 1361 /* 1362 * We need extra memory allocated through BOP_ALLOC. 1363 */ 1364 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1365 alloc_alignsize); 1366 1367 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1368 1369 bop_base = (uintptr_t)alloc_base; 1370 1371 alloc_base += alloc_sz; 1372 1373 if (page_hash == NULL) { 1374 page_hash = (struct page **)bop_base; 1375 bop_base = roundup(bop_base + pagehash_sz, 1376 ecache_alignsize); 1377 } 1378 1379 if (ctrs_base == NULL) { 1380 ctrs_base = (caddr_t)bop_base; 1381 bop_base = roundup(bop_base + ctrs_sz, 1382 ecache_alignsize); 1383 } 1384 1385 if (memseg_base == NULL) { 1386 memseg_base = (struct memseg *)bop_base; 1387 bop_base = roundup(bop_base + memseg_sz, 1388 ecache_alignsize); 1389 } 1390 1391 if (kpm_enable && kpm_pp_base == NULL) { 1392 kpm_pp_base = (uintptr_t)bop_base; 1393 bop_base = roundup(bop_base + kpm_pp_sz, 1394 ecache_alignsize); 1395 } 1396 1397 ASSERT(bop_base <= (uintptr_t)alloc_base); 1398 } 1399 1400 PRM_DEBUG(page_hash); 1401 PRM_DEBUG(memseg_base); 1402 PRM_DEBUG(kpm_pp_base); 1403 PRM_DEBUG(kpm_pp_sz); 1404 PRM_DEBUG(pp_base); 1405 PRM_DEBUG(pp_sz); 1406 PRM_DEBUG(alloc_base); 1407 1408 #ifdef TRAPTRACE 1409 alloc_base = trap_trace_alloc(alloc_base); 1410 PRM_DEBUG(alloc_base); 1411 #endif /* TRAPTRACE */ 1412 1413 /* 1414 * In theory it's possible that kmem64 chunk is 0 sized 1415 * (on very small machines). Check for that. 1416 */ 1417 if (alloc_base == kmem64_base) { 1418 kmem64_base = NULL; 1419 kmem64_end = NULL; 1420 kmem64_aligned_end = NULL; 1421 goto kmem64_alloced; 1422 } 1423 1424 /* 1425 * Allocate kmem64 memory. 1426 * Round up to end of large page and overmap. 1427 * kmem64_end..kmem64_aligned_end is added to memory list for reuse 1428 */ 1429 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1430 MMU_PAGESIZE); 1431 1432 /* 1433 * Make one large memory alloc after figuring out the 64-bit size. This 1434 * will enable use of the largest page size appropriate for the system 1435 * architecture. 1436 */ 1437 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 1438 ASSERT(IS_P2ALIGNED(kmem64_base, TTEBYTES(max_bootlp_tteszc))); 1439 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 1440 size_t asize; 1441 #if !defined(C_OBP) 1442 unsigned long long pa; 1443 #endif /* !C_OBP */ 1444 1445 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 1446 continue; 1447 kmem64_alignsize = TTEBYTES(i); 1448 kmem64_szc = i; 1449 1450 /* limit page size for small memory */ 1451 if (mmu_btop(kmem64_alignsize) > (npages >> 2)) 1452 continue; 1453 1454 kmem64_aligned_end = (caddr_t)roundup((uintptr_t)kmem64_end, 1455 kmem64_alignsize); 1456 asize = kmem64_aligned_end - kmem64_base; 1457 #if !defined(C_OBP) 1458 if (prom_allocate_phys(asize, kmem64_alignsize, &pa) == 0) { 1459 if (prom_claim_virt(asize, kmem64_base) != 1460 (caddr_t)-1) { 1461 kmem64_pabase = pa; 1462 install_kmem64_tte(); 1463 break; 1464 } else { 1465 prom_free_phys(asize, pa); 1466 } 1467 } 1468 #else /* !C_OBP */ 1469 if ((caddr_t)BOP_ALLOC(bootops, kmem64_base, asize, 1470 kmem64_alignsize) == kmem64_base) { 1471 kmem64_pabase = va_to_pa(kmem64_base); 1472 break; 1473 } 1474 #endif /* !C_OBP */ 1475 if (i == TTE8K) { 1476 prom_panic("kmem64 allocation failure"); 1477 } 1478 } 1479 1480 PRM_DEBUG(kmem64_base); 1481 PRM_DEBUG(kmem64_end); 1482 PRM_DEBUG(kmem64_aligned_end); 1483 PRM_DEBUG(kmem64_alignsize); 1484 1485 /* 1486 * Now set pa using saved va from above. 1487 */ 1488 if (&ecache_init_scrub_flush_area) { 1489 (void) ecache_init_scrub_flush_area(NULL); 1490 } 1491 1492 kmem64_alloced: 1493 1494 /* 1495 * Initialize per page size free list counters. 1496 */ 1497 ctrs_end = page_ctrs_alloc(ctrs_base); 1498 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1499 1500 /* 1501 * Allocate space for the interrupt vector table and also for the 1502 * reserved interrupt vector data structures. 1503 */ 1504 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 1505 IVSIZE, MMU_PAGESIZE); 1506 if (memspace != (caddr_t)intr_vec_table) 1507 prom_panic("interrupt vector table allocation failure"); 1508 1509 /* 1510 * The memory lists from boot are allocated from the heap arena 1511 * so that later they can be freed and/or reallocated. 1512 */ 1513 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1514 prom_panic("could not retrieve property \"extent\""); 1515 1516 /* 1517 * Between now and when we finish copying in the memory lists, 1518 * allocations happen so the space gets fragmented and the 1519 * lists longer. Leave enough space for lists twice as long 1520 * as what boot says it has now; roundup to a pagesize. 1521 * Also add space for the final phys-avail copy in the fixup 1522 * routine. 1523 */ 1524 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1525 roundup(IVSIZE, MMU_PAGESIZE)); 1526 memlist_sz *= 4; 1527 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1528 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1529 if (memspace == NULL) 1530 halt("Boot allocation failed."); 1531 1532 memlist = (struct memlist *)memspace; 1533 memlist_end = (char *)memspace + memlist_sz; 1534 1535 PRM_DEBUG(memlist); 1536 PRM_DEBUG(memlist_end); 1537 PRM_DEBUG(sysbase); 1538 PRM_DEBUG(syslimit); 1539 1540 kernelheap_init((void *)sysbase, (void *)syslimit, 1541 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1542 1543 /* 1544 * Take the most current snapshot we can by calling mem-update. 1545 */ 1546 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1547 &boot_physavail, &boot_physavail_len, 1548 &boot_virtavail, &boot_virtavail_len); 1549 1550 /* 1551 * Remove the space used by BOP_ALLOC from the kernel heap 1552 * plus the area actually used by the OBP (if any) 1553 * ignoring virtual addresses in virt_avail, above syslimit. 1554 */ 1555 virt_avail = memlist; 1556 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1557 1558 for (cur = virt_avail; cur->next; cur = cur->next) { 1559 uint64_t range_base, range_size; 1560 1561 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1562 continue; 1563 if (range_base >= (uint64_t)syslimit) 1564 break; 1565 /* 1566 * Limit the range to end at syslimit. 1567 */ 1568 range_size = MIN(cur->next->address, 1569 (uint64_t)syslimit) - range_base; 1570 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1571 0, 0, (void *)range_base, (void *)(range_base + range_size), 1572 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1573 } 1574 1575 phys_avail = memlist; 1576 (void) copy_physavail(boot_physavail, boot_physavail_len, 1577 &memlist, 0, 0); 1578 1579 /* 1580 * Add any unused kmem64 memory from overmapped page 1581 * (Note: va_to_pa does not work for kmem64_end) 1582 */ 1583 if (kmem64_end < kmem64_aligned_end) { 1584 uint64_t overlap_size = kmem64_aligned_end - kmem64_end; 1585 uint64_t overlap_pa = kmem64_pabase + 1586 (kmem64_end - kmem64_base); 1587 1588 PRM_DEBUG(overlap_pa); 1589 PRM_DEBUG(overlap_size); 1590 memlist_add(overlap_pa, overlap_size, &memlist, &phys_avail); 1591 } 1592 1593 /* 1594 * Add any extra memory after e_text to the phys_avail list, as long 1595 * as there's at least a page to add. 1596 */ 1597 if (extra_etpg) 1598 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1599 &memlist, &phys_avail); 1600 1601 /* 1602 * Add any extra memory at the end of the ndata region if there's at 1603 * least a page to add. There might be a few more pages available in 1604 * the middle of the ndata region, but for now they are ignored. 1605 */ 1606 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1607 if (nalloc_base == NULL) 1608 nalloc_base = nalloc_end; 1609 ndata_remain_sz = nalloc_end - nalloc_base; 1610 1611 if (ndata_remain_sz >= MMU_PAGESIZE) 1612 memlist_add(va_to_pa(nalloc_base), 1613 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1614 1615 PRM_DEBUG(memlist); 1616 PRM_DEBUG(memlist_sz); 1617 PRM_DEBUG(memspace); 1618 1619 if ((caddr_t)memlist > (memspace + memlist_sz)) 1620 prom_panic("memlist overflow"); 1621 1622 PRM_DEBUG(pp_base); 1623 PRM_DEBUG(memseg_base); 1624 PRM_DEBUG(npages); 1625 1626 /* 1627 * Initialize the page structures from the memory lists. 1628 */ 1629 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1630 1631 availrmem_initial = availrmem = freemem; 1632 PRM_DEBUG(availrmem); 1633 1634 /* 1635 * Some of the locks depend on page_hashsz being set! 1636 * kmem_init() depends on this; so, keep it here. 1637 */ 1638 page_lock_init(); 1639 1640 /* 1641 * Initialize kernel memory allocator. 1642 */ 1643 kmem_init(); 1644 1645 /* 1646 * Factor in colorequiv to check additional 'equivalent' bins 1647 */ 1648 if (&page_set_colorequiv_arr_cpu != NULL) 1649 page_set_colorequiv_arr_cpu(); 1650 else 1651 page_set_colorequiv_arr(); 1652 1653 /* 1654 * Initialize bp_mapin(). 1655 */ 1656 bp_init(shm_alignment, HAT_STRICTORDER); 1657 1658 /* 1659 * Reserve space for panicbuf, intr_vec_table and reserved interrupt 1660 * vector data structures from the 32-bit heap. 1661 */ 1662 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1663 panicbuf, panicbuf + PANICBUFSIZE, 1664 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1665 1666 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1667 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1668 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1669 1670 mem_config_init(); 1671 } 1672 1673 static void 1674 startup_modules(void) 1675 { 1676 int proplen, nhblk1, nhblk8; 1677 size_t nhblksz; 1678 pgcnt_t pages_per_hblk; 1679 size_t hme8blk_sz, hme1blk_sz; 1680 1681 /* 1682 * Log any optional messages from the boot program 1683 */ 1684 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1685 if (proplen > 0) { 1686 char *msg; 1687 size_t len = (size_t)proplen; 1688 1689 msg = kmem_zalloc(len, KM_SLEEP); 1690 (void) BOP_GETPROP(bootops, "boot-message", msg); 1691 cmn_err(CE_CONT, "?%s\n", msg); 1692 kmem_free(msg, len); 1693 } 1694 1695 /* 1696 * Let the platforms have a chance to change default 1697 * values before reading system file. 1698 */ 1699 if (&set_platform_defaults) 1700 set_platform_defaults(); 1701 1702 /* 1703 * Calculate default settings of system parameters based upon 1704 * maxusers, yet allow to be overridden via the /etc/system file. 1705 */ 1706 param_calc(0); 1707 1708 mod_setup(); 1709 1710 /* 1711 * If this is a positron, complain and halt. 1712 */ 1713 if (&iam_positron && iam_positron()) { 1714 cmn_err(CE_WARN, "This hardware platform is not supported" 1715 " by this release of Solaris.\n"); 1716 #ifdef DEBUG 1717 prom_enter_mon(); /* Type 'go' to resume */ 1718 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1719 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1720 1721 #else /* DEBUG */ 1722 halt(0); 1723 #endif /* DEBUG */ 1724 } 1725 1726 /* 1727 * If we are running firmware that isn't 64-bit ready 1728 * then complain and halt. 1729 */ 1730 do_prom_version_check(); 1731 1732 /* 1733 * Initialize system parameters 1734 */ 1735 param_init(); 1736 1737 /* 1738 * maxmem is the amount of physical memory we're playing with. 1739 */ 1740 maxmem = physmem; 1741 1742 /* Set segkp limits. */ 1743 ncbase = kdi_segdebugbase; 1744 ncend = kdi_segdebugbase; 1745 1746 /* 1747 * Initialize the hat layer. 1748 */ 1749 hat_init(); 1750 1751 /* 1752 * Initialize segment management stuff. 1753 */ 1754 seg_init(); 1755 1756 /* 1757 * Create the va>tte handler, so the prom can understand 1758 * kernel translations. The handler is installed later, just 1759 * as we are about to take over the trap table from the prom. 1760 */ 1761 create_va_to_tte(); 1762 1763 /* 1764 * Load the forthdebugger (optional) 1765 */ 1766 forthdebug_init(); 1767 1768 /* 1769 * Create OBP node for console input callbacks 1770 * if it is needed. 1771 */ 1772 startup_create_io_node(); 1773 1774 if (modloadonly("fs", "specfs") == -1) 1775 halt("Can't load specfs"); 1776 1777 if (modloadonly("fs", "devfs") == -1) 1778 halt("Can't load devfs"); 1779 1780 if (modloadonly("misc", "swapgeneric") == -1) 1781 halt("Can't load swapgeneric"); 1782 1783 (void) modloadonly("sys", "lbl_edition"); 1784 1785 dispinit(); 1786 1787 /* 1788 * Infer meanings to the members of the idprom buffer. 1789 */ 1790 parse_idprom(); 1791 1792 /* Read cluster configuration data. */ 1793 clconf_init(); 1794 1795 setup_ddi(); 1796 1797 /* 1798 * Lets take this opportunity to load the root device. 1799 */ 1800 if (loadrootmodules() != 0) 1801 debug_enter("Can't load the root filesystem"); 1802 1803 /* 1804 * Load tod driver module for the tod part found on this system. 1805 * Recompute the cpu frequency/delays based on tod as tod part 1806 * tends to keep time more accurately. 1807 */ 1808 if (&load_tod_module) 1809 load_tod_module(); 1810 1811 /* 1812 * Allow platforms to load modules which might 1813 * be needed after bootops are gone. 1814 */ 1815 if (&load_platform_modules) 1816 load_platform_modules(); 1817 1818 setcpudelay(); 1819 1820 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1821 &boot_physavail, &boot_physavail_len, 1822 &boot_virtavail, &boot_virtavail_len); 1823 1824 /* 1825 * Calculation and allocation of hmeblks needed to remap 1826 * the memory allocated by PROM till now. 1827 * Overestimate the number of hblk1 elements by assuming 1828 * worst case of TTE64K mappings. 1829 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1830 */ 1831 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1832 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1833 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1834 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1835 1836 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1837 1838 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1839 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1840 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1841 1842 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1843 1844 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1845 nhblk8 = 0; 1846 while (bop_alloc_pages > 1) { 1847 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1848 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1849 bop_alloc_pages *= hme8blk_sz; 1850 bop_alloc_pages = btopr(bop_alloc_pages); 1851 } 1852 nhblk8 += 2; 1853 1854 /* 1855 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1856 * boundary, the number of hblk8's needed to map the entries in the 1857 * boot_virtavail list needs to be adjusted to take this into 1858 * consideration. Thus, we need to add additional hblk8's since it 1859 * is possible that an hblk8 will not have all 8 slots used due to 1860 * alignment constraints. Since there were boot_virtavail_len entries 1861 * in that list, we need to add that many hblk8's to the number 1862 * already calculated to make sure we don't underestimate. 1863 */ 1864 nhblk8 += boot_virtavail_len; 1865 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1866 1867 /* Allocate in pagesize chunks */ 1868 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1869 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1870 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1871 } 1872 1873 static void 1874 startup_bop_gone(void) 1875 { 1876 extern int bop_io_quiesced; 1877 1878 /* 1879 * Destroy the MD initialized at startup 1880 * The startup initializes the MD framework 1881 * using prom and BOP alloc free it now. 1882 */ 1883 mach_descrip_startup_fini(); 1884 1885 /* 1886 * Call back into boot and release boots resources. 1887 */ 1888 BOP_QUIESCE_IO(bootops); 1889 bop_io_quiesced = 1; 1890 1891 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1892 &boot_physavail, &boot_physavail_len, 1893 &boot_virtavail, &boot_virtavail_len); 1894 /* 1895 * Copy physinstalled list into kernel space. 1896 */ 1897 phys_install = memlist; 1898 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1899 1900 /* 1901 * setup physically contiguous area twice as large as the ecache. 1902 * this is used while doing displacement flush of ecaches 1903 */ 1904 if (&ecache_flush_address) { 1905 ecache_flushaddr = ecache_flush_address(); 1906 if (ecache_flushaddr == (uint64_t)-1) { 1907 cmn_err(CE_PANIC, 1908 "startup: no memory to set ecache_flushaddr"); 1909 } 1910 } 1911 1912 /* 1913 * Virtual available next. 1914 */ 1915 ASSERT(virt_avail != NULL); 1916 memlist_free_list(virt_avail); 1917 virt_avail = memlist; 1918 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1919 1920 /* 1921 * Last chance to ask our booter questions .. 1922 */ 1923 } 1924 1925 1926 /* 1927 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1928 * allocations have been performed. We can't call it in startup_bop_gone 1929 * since later operations can cause obp to allocate more memory. 1930 */ 1931 void 1932 startup_fixup_physavail(void) 1933 { 1934 struct memlist *cur; 1935 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1936 1937 PRM_DEBUG(kmem64_overmap_size); 1938 1939 /* 1940 * take the most current snapshot we can by calling mem-update 1941 */ 1942 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1943 &boot_physavail, &boot_physavail_len, 1944 &boot_virtavail, &boot_virtavail_len); 1945 1946 /* 1947 * Copy phys_avail list, again. 1948 * Both the kernel/boot and the prom have been allocating 1949 * from the original list we copied earlier. 1950 */ 1951 cur = memlist; 1952 (void) copy_physavail(boot_physavail, boot_physavail_len, 1953 &memlist, 0, 0); 1954 1955 /* 1956 * Add any unused kmem64 memory from overmapped page 1957 * (Note: va_to_pa does not work for kmem64_end) 1958 */ 1959 if (kmem64_overmap_size) { 1960 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1961 kmem64_overmap_size, 1962 &memlist, &cur); 1963 } 1964 1965 /* 1966 * Add any extra memory after e_text we added to the phys_avail list 1967 * back to the old list. 1968 */ 1969 if (extra_etpg) 1970 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1971 &memlist, &cur); 1972 if (ndata_remain_sz >= MMU_PAGESIZE) 1973 memlist_add(va_to_pa(nalloc_base), 1974 (uint64_t)ndata_remain_sz, &memlist, &cur); 1975 1976 /* 1977 * There isn't any bounds checking on the memlist area 1978 * so ensure it hasn't overgrown. 1979 */ 1980 if ((caddr_t)memlist > (caddr_t)memlist_end) 1981 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1982 1983 /* 1984 * The kernel removes the pages that were allocated for it from 1985 * the freelist, but we now have to find any -extra- pages that 1986 * the prom has allocated for it's own book-keeping, and remove 1987 * them from the freelist too. sigh. 1988 */ 1989 fix_prom_pages(phys_avail, cur); 1990 1991 ASSERT(phys_avail != NULL); 1992 memlist_free_list(phys_avail); 1993 phys_avail = cur; 1994 1995 /* 1996 * We're done with boot. Just after this point in time, boot 1997 * gets unmapped, so we can no longer rely on its services. 1998 * Zero the bootops to indicate this fact. 1999 */ 2000 bootops = (struct bootops *)NULL; 2001 BOOTOPS_GONE(); 2002 } 2003 2004 static void 2005 startup_vm(void) 2006 { 2007 size_t i; 2008 struct segmap_crargs a; 2009 struct segkpm_crargs b; 2010 2011 uint64_t avmem; 2012 caddr_t va; 2013 pgcnt_t max_phys_segkp; 2014 int mnode; 2015 2016 extern int use_brk_lpg, use_stk_lpg; 2017 2018 /* 2019 * get prom's mappings, create hments for them and switch 2020 * to the kernel context. 2021 */ 2022 hat_kern_setup(); 2023 2024 /* 2025 * Take over trap table 2026 */ 2027 setup_trap_table(); 2028 2029 /* 2030 * Install the va>tte handler, so that the prom can handle 2031 * misses and understand the kernel table layout in case 2032 * we need call into the prom. 2033 */ 2034 install_va_to_tte(); 2035 2036 /* 2037 * Set a flag to indicate that the tba has been taken over. 2038 */ 2039 tba_taken_over = 1; 2040 2041 /* initialize MMU primary context register */ 2042 mmu_init_kcontext(); 2043 2044 /* 2045 * The boot cpu can now take interrupts, x-calls, x-traps 2046 */ 2047 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 2048 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 2049 2050 /* 2051 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 2052 */ 2053 tbr_wr_addr_inited = 1; 2054 2055 /* 2056 * Initialize VM system, and map kernel address space. 2057 */ 2058 kvm_init(); 2059 2060 /* 2061 * XXX4U: previously, we initialized and turned on 2062 * the caches at this point. But of course we have 2063 * nothing to do, as the prom has already done this 2064 * for us -- main memory must be E$able at all times. 2065 */ 2066 2067 /* 2068 * If the following is true, someone has patched 2069 * phsymem to be less than the number of pages that 2070 * the system actually has. Remove pages until system 2071 * memory is limited to the requested amount. Since we 2072 * have allocated page structures for all pages, we 2073 * correct the amount of memory we want to remove 2074 * by the size of the memory used to hold page structures 2075 * for the non-used pages. 2076 */ 2077 if (physmem < npages) { 2078 pgcnt_t diff, off; 2079 struct page *pp; 2080 struct seg kseg; 2081 2082 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 2083 2084 off = 0; 2085 diff = npages - physmem; 2086 diff -= mmu_btopr(diff * sizeof (struct page)); 2087 kseg.s_as = &kas; 2088 while (diff--) { 2089 pp = page_create_va(&unused_pages_vp, (offset_t)off, 2090 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 2091 &kseg, (caddr_t)off); 2092 if (pp == NULL) 2093 cmn_err(CE_PANIC, "limited physmem too much!"); 2094 page_io_unlock(pp); 2095 page_downgrade(pp); 2096 availrmem--; 2097 off += MMU_PAGESIZE; 2098 } 2099 } 2100 2101 /* 2102 * When printing memory, show the total as physmem less 2103 * that stolen by a debugger. 2104 */ 2105 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 2106 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 2107 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 2108 2109 avmem = (uint64_t)freemem << PAGESHIFT; 2110 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 2111 2112 /* 2113 * For small memory systems disable automatic large pages. 2114 */ 2115 if (physmem < privm_lpg_min_physmem) { 2116 use_brk_lpg = 0; 2117 use_stk_lpg = 0; 2118 } 2119 2120 /* 2121 * Perform platform specific freelist processing 2122 */ 2123 if (&plat_freelist_process) { 2124 for (mnode = 0; mnode < max_mem_nodes; mnode++) 2125 if (mem_node_config[mnode].exists) 2126 plat_freelist_process(mnode); 2127 } 2128 2129 /* 2130 * Initialize the segkp segment type. We position it 2131 * after the configured tables and buffers (whose end 2132 * is given by econtig) and before V_WKBASE_ADDR. 2133 * Also in this area is segkmap (size SEGMAPSIZE). 2134 */ 2135 2136 /* XXX - cache alignment? */ 2137 va = (caddr_t)SEGKPBASE; 2138 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2139 2140 max_phys_segkp = (physmem * 2); 2141 2142 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2143 segkpsize = btop(SEGKPDEFSIZE); 2144 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2145 "segkpsize has been reset to %ld pages", segkpsize); 2146 } 2147 2148 i = ptob(MIN(segkpsize, max_phys_segkp)); 2149 2150 rw_enter(&kas.a_lock, RW_WRITER); 2151 if (seg_attach(&kas, va, i, segkp) < 0) 2152 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2153 if (segkp_create(segkp) != 0) 2154 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2155 rw_exit(&kas.a_lock); 2156 2157 /* 2158 * kpm segment 2159 */ 2160 segmap_kpm = kpm_enable && 2161 segmap_kpm && PAGESIZE == MAXBSIZE; 2162 2163 if (kpm_enable) { 2164 rw_enter(&kas.a_lock, RW_WRITER); 2165 2166 /* 2167 * The segkpm virtual range range is larger than the 2168 * actual physical memory size and also covers gaps in 2169 * the physical address range for the following reasons: 2170 * . keep conversion between segkpm and physical addresses 2171 * simple, cheap and unambiguous. 2172 * . avoid extension/shrink of the the segkpm in case of DR. 2173 * . avoid complexity for handling of virtual addressed 2174 * caches, segkpm and the regular mapping scheme must be 2175 * kept in sync wrt. the virtual color of mapped pages. 2176 * Any accesses to virtual segkpm ranges not backed by 2177 * physical memory will fall through the memseg pfn hash 2178 * and will be handled in segkpm_fault. 2179 * Additional kpm_size spaces needed for vac alias prevention. 2180 */ 2181 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2182 segkpm) < 0) 2183 cmn_err(CE_PANIC, "cannot attach segkpm"); 2184 2185 b.prot = PROT_READ | PROT_WRITE; 2186 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2187 2188 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2189 panic("segkpm_create segkpm"); 2190 2191 rw_exit(&kas.a_lock); 2192 2193 mach_kpm_init(); 2194 } 2195 2196 if (!segzio_fromheap) { 2197 size_t size; 2198 size_t physmem_b = mmu_ptob(physmem); 2199 2200 /* size is in bytes, segziosize is in pages */ 2201 if (segziosize == 0) { 2202 size = physmem_b; 2203 } else { 2204 size = mmu_ptob(segziosize); 2205 } 2206 2207 if (size < SEGZIOMINSIZE) { 2208 size = SEGZIOMINSIZE; 2209 } else if (size > SEGZIOMAXSIZE) { 2210 size = SEGZIOMAXSIZE; 2211 /* 2212 * On 64-bit x86, we only have 2TB of KVA. This exists 2213 * for parity with x86. 2214 * 2215 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2216 * doesn't consume all of KVA. However, if we have a 2217 * system that has more thant 512gb of physical memory, 2218 * we can actually consume about half of the difference 2219 * between 512gb and the rest of the available physical 2220 * memory. 2221 */ 2222 if (physmem_b > SEGZIOMAXSIZE) { 2223 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2224 } 2225 } 2226 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2227 /* put the base of the ZIO segment after the kpm segment */ 2228 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2229 PRM_DEBUG(segziosize); 2230 PRM_DEBUG(segzio_base); 2231 2232 /* 2233 * On some platforms, kvm_init is called after the kpm 2234 * sizes have been determined. On SPARC, kvm_init is called 2235 * before, so we have to attach the kzioseg after kvm is 2236 * initialized, otherwise we'll try to allocate from the boot 2237 * area since the kernel heap hasn't yet been configured. 2238 */ 2239 rw_enter(&kas.a_lock, RW_WRITER); 2240 2241 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2242 &kzioseg); 2243 (void) segkmem_zio_create(&kzioseg); 2244 2245 /* create zio area covering new segment */ 2246 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2247 2248 rw_exit(&kas.a_lock); 2249 } 2250 2251 2252 /* 2253 * Now create generic mapping segment. This mapping 2254 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2255 * virtual address is greater than the amount of free 2256 * memory that is available, then we trim back the 2257 * segment size to that amount 2258 */ 2259 va = (caddr_t)SEGMAPBASE; 2260 2261 /* 2262 * 1201049: segkmap base address must be MAXBSIZE aligned 2263 */ 2264 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2265 2266 /* 2267 * Set size of segmap to percentage of freemem at boot, 2268 * but stay within the allowable range 2269 * Note we take percentage before converting from pages 2270 * to bytes to avoid an overflow on 32-bit kernels. 2271 */ 2272 i = mmu_ptob((freemem * segmap_percent) / 100); 2273 2274 if (i < MINMAPSIZE) 2275 i = MINMAPSIZE; 2276 2277 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2278 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2279 2280 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2281 2282 rw_enter(&kas.a_lock, RW_WRITER); 2283 if (seg_attach(&kas, va, i, segkmap) < 0) 2284 cmn_err(CE_PANIC, "cannot attach segkmap"); 2285 2286 a.prot = PROT_READ | PROT_WRITE; 2287 a.shmsize = shm_alignment; 2288 a.nfreelist = 0; /* use segmap driver defaults */ 2289 2290 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2291 panic("segmap_create segkmap"); 2292 rw_exit(&kas.a_lock); 2293 2294 segdev_init(); 2295 } 2296 2297 static void 2298 startup_end(void) 2299 { 2300 if ((caddr_t)memlist > (caddr_t)memlist_end) 2301 panic("memlist overflow 2"); 2302 memlist_free_block((caddr_t)memlist, 2303 ((caddr_t)memlist_end - (caddr_t)memlist)); 2304 memlist = NULL; 2305 2306 /* enable page_relocation since OBP is now done */ 2307 page_relocate_ready = 1; 2308 2309 /* 2310 * Perform tasks that get done after most of the VM 2311 * initialization has been done but before the clock 2312 * and other devices get started. 2313 */ 2314 kern_setup1(); 2315 2316 /* 2317 * Intialize the VM arenas for allocating physically 2318 * contiguus memory chunk for interrupt queues snd 2319 * allocate/register boot cpu's queues, if any and 2320 * allocate dump buffer for sun4v systems to store 2321 * extra crash information during crash dump 2322 */ 2323 contig_mem_init(); 2324 mach_descrip_init(); 2325 2326 if (cpu_intrq_setup(CPU)) { 2327 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2328 } 2329 cpu_intrq_register(CPU); 2330 mach_htraptrace_setup(CPU->cpu_id); 2331 mach_htraptrace_configure(CPU->cpu_id); 2332 mach_dump_buffer_init(); 2333 2334 /* 2335 * Initialize interrupt related stuff 2336 */ 2337 cpu_intr_alloc(CPU, NINTR_THREADS); 2338 2339 (void) splzs(); /* allow hi clock ints but not zs */ 2340 2341 /* 2342 * Initialize errors. 2343 */ 2344 error_init(); 2345 2346 /* 2347 * Note that we may have already used kernel bcopy before this 2348 * point - but if you really care about this, adb the use_hw_* 2349 * variables to 0 before rebooting. 2350 */ 2351 mach_hw_copy_limit(); 2352 2353 /* 2354 * Install the "real" preemption guards before DDI services 2355 * are available. 2356 */ 2357 (void) prom_set_preprom(kern_preprom); 2358 (void) prom_set_postprom(kern_postprom); 2359 CPU->cpu_m.mutex_ready = 1; 2360 2361 /* 2362 * Initialize segnf (kernel support for non-faulting loads). 2363 */ 2364 segnf_init(); 2365 2366 /* 2367 * Configure the root devinfo node. 2368 */ 2369 configure(); /* set up devices */ 2370 mach_cpu_halt_idle(); 2371 } 2372 2373 2374 void 2375 post_startup(void) 2376 { 2377 #ifdef PTL1_PANIC_DEBUG 2378 extern void init_ptl1_thread(void); 2379 #endif /* PTL1_PANIC_DEBUG */ 2380 extern void abort_sequence_init(void); 2381 2382 /* 2383 * Set the system wide, processor-specific flags to be passed 2384 * to userland via the aux vector for performance hints and 2385 * instruction set extensions. 2386 */ 2387 bind_hwcap(); 2388 2389 /* 2390 * Startup memory scrubber (if any) 2391 */ 2392 mach_memscrub(); 2393 2394 /* 2395 * Allocate soft interrupt to handle abort sequence. 2396 */ 2397 abort_sequence_init(); 2398 2399 /* 2400 * Configure the rest of the system. 2401 * Perform forceloading tasks for /etc/system. 2402 */ 2403 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2404 /* 2405 * ON4.0: Force /proc module in until clock interrupt handle fixed 2406 * ON4.0: This must be fixed or restated in /etc/systems. 2407 */ 2408 (void) modload("fs", "procfs"); 2409 2410 /* load machine class specific drivers */ 2411 load_mach_drivers(); 2412 2413 /* load platform specific drivers */ 2414 if (&load_platform_drivers) 2415 load_platform_drivers(); 2416 2417 /* load vis simulation module, if we are running w/fpu off */ 2418 if (!fpu_exists) { 2419 if (modload("misc", "vis") == -1) 2420 halt("Can't load vis"); 2421 } 2422 2423 mach_fpras(); 2424 2425 maxmem = freemem; 2426 2427 #ifdef PTL1_PANIC_DEBUG 2428 init_ptl1_thread(); 2429 #endif /* PTL1_PANIC_DEBUG */ 2430 2431 if (&cif_init) 2432 cif_init(); 2433 } 2434 2435 #ifdef PTL1_PANIC_DEBUG 2436 int ptl1_panic_test = 0; 2437 int ptl1_panic_xc_one_test = 0; 2438 int ptl1_panic_xc_all_test = 0; 2439 int ptl1_panic_xt_one_test = 0; 2440 int ptl1_panic_xt_all_test = 0; 2441 kthread_id_t ptl1_thread_p = NULL; 2442 kcondvar_t ptl1_cv; 2443 kmutex_t ptl1_mutex; 2444 int ptl1_recurse_count_threshold = 0x40; 2445 int ptl1_recurse_trap_threshold = 0x3d; 2446 extern void ptl1_recurse(int, int); 2447 extern void ptl1_panic_xt(int, int); 2448 2449 /* 2450 * Called once per second by timeout() to wake up 2451 * the ptl1_panic thread to see if it should cause 2452 * a trap to the ptl1_panic() code. 2453 */ 2454 /* ARGSUSED */ 2455 static void 2456 ptl1_wakeup(void *arg) 2457 { 2458 mutex_enter(&ptl1_mutex); 2459 cv_signal(&ptl1_cv); 2460 mutex_exit(&ptl1_mutex); 2461 } 2462 2463 /* 2464 * ptl1_panic cross call function: 2465 * Needed because xc_one() and xc_some() can pass 2466 * 64 bit args but ptl1_recurse() expects ints. 2467 */ 2468 static void 2469 ptl1_panic_xc(void) 2470 { 2471 ptl1_recurse(ptl1_recurse_count_threshold, 2472 ptl1_recurse_trap_threshold); 2473 } 2474 2475 /* 2476 * The ptl1 thread waits for a global flag to be set 2477 * and uses the recurse thresholds to set the stack depth 2478 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2479 * or indirectly via the cross call and cross trap functions. 2480 * 2481 * This is useful testing stack overflows and normal 2482 * ptl1_panic() states with a know stack frame. 2483 * 2484 * ptl1_recurse() is an asm function in ptl1_panic.s that 2485 * sets the {In, Local, Out, and Global} registers to a 2486 * know state on the stack and just prior to causing a 2487 * test ptl1_panic trap. 2488 */ 2489 static void 2490 ptl1_thread(void) 2491 { 2492 mutex_enter(&ptl1_mutex); 2493 while (ptl1_thread_p) { 2494 cpuset_t other_cpus; 2495 int cpu_id; 2496 int my_cpu_id; 2497 int target_cpu_id; 2498 int target_found; 2499 2500 if (ptl1_panic_test) { 2501 ptl1_recurse(ptl1_recurse_count_threshold, 2502 ptl1_recurse_trap_threshold); 2503 } 2504 2505 /* 2506 * Find potential targets for x-call and x-trap, 2507 * if any exist while preempt is disabled we 2508 * start a ptl1_panic if requested via a 2509 * globals. 2510 */ 2511 kpreempt_disable(); 2512 my_cpu_id = CPU->cpu_id; 2513 other_cpus = cpu_ready_set; 2514 CPUSET_DEL(other_cpus, CPU->cpu_id); 2515 target_found = 0; 2516 if (!CPUSET_ISNULL(other_cpus)) { 2517 /* 2518 * Pick the first one 2519 */ 2520 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2521 if (cpu_id == my_cpu_id) 2522 continue; 2523 2524 if (CPU_XCALL_READY(cpu_id)) { 2525 target_cpu_id = cpu_id; 2526 target_found = 1; 2527 break; 2528 } 2529 } 2530 ASSERT(target_found); 2531 2532 if (ptl1_panic_xc_one_test) { 2533 xc_one(target_cpu_id, 2534 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2535 } 2536 if (ptl1_panic_xc_all_test) { 2537 xc_some(other_cpus, 2538 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2539 } 2540 if (ptl1_panic_xt_one_test) { 2541 xt_one(target_cpu_id, 2542 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2543 } 2544 if (ptl1_panic_xt_all_test) { 2545 xt_some(other_cpus, 2546 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2547 } 2548 } 2549 kpreempt_enable(); 2550 (void) timeout(ptl1_wakeup, NULL, hz); 2551 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2552 } 2553 mutex_exit(&ptl1_mutex); 2554 } 2555 2556 /* 2557 * Called during early startup to create the ptl1_thread 2558 */ 2559 void 2560 init_ptl1_thread(void) 2561 { 2562 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2563 &p0, TS_RUN, 0); 2564 } 2565 #endif /* PTL1_PANIC_DEBUG */ 2566 2567 2568 /* 2569 * Add to a memory list. 2570 * start = start of new memory segment 2571 * len = length of new memory segment in bytes 2572 * memlistp = pointer to array of available memory segment structures 2573 * curmemlistp = memory list to which to add segment. 2574 */ 2575 static void 2576 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2577 struct memlist **curmemlistp) 2578 { 2579 struct memlist *new; 2580 2581 new = *memlistp; 2582 new->address = start; 2583 new->size = len; 2584 *memlistp = new + 1; 2585 2586 memlist_insert(new, curmemlistp); 2587 } 2588 2589 /* 2590 * In the case of architectures that support dynamic addition of 2591 * memory at run-time there are two cases where memsegs need to 2592 * be initialized and added to the memseg list. 2593 * 1) memsegs that are constructed at startup. 2594 * 2) memsegs that are constructed at run-time on 2595 * hot-plug capable architectures. 2596 * This code was originally part of the function kphysm_init(). 2597 */ 2598 2599 static void 2600 memseg_list_add(struct memseg *memsegp) 2601 { 2602 struct memseg **prev_memsegp; 2603 pgcnt_t num; 2604 2605 /* insert in memseg list, decreasing number of pages order */ 2606 2607 num = MSEG_NPAGES(memsegp); 2608 2609 for (prev_memsegp = &memsegs; *prev_memsegp; 2610 prev_memsegp = &((*prev_memsegp)->next)) { 2611 if (num > MSEG_NPAGES(*prev_memsegp)) 2612 break; 2613 } 2614 2615 memsegp->next = *prev_memsegp; 2616 *prev_memsegp = memsegp; 2617 2618 if (kpm_enable) { 2619 memsegp->nextpa = (memsegp->next) ? 2620 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2621 2622 if (prev_memsegp != &memsegs) { 2623 struct memseg *msp; 2624 msp = (struct memseg *)((caddr_t)prev_memsegp - 2625 offsetof(struct memseg, next)); 2626 msp->nextpa = va_to_pa(memsegp); 2627 } else { 2628 memsegspa = va_to_pa(memsegs); 2629 } 2630 } 2631 } 2632 2633 /* 2634 * PSM add_physmem_cb(). US-II and newer processors have some 2635 * flavor of the prefetch capability implemented. We exploit 2636 * this capability for optimum performance. 2637 */ 2638 #define PREFETCH_BYTES 64 2639 2640 void 2641 add_physmem_cb(page_t *pp, pfn_t pnum) 2642 { 2643 extern void prefetch_page_w(void *); 2644 2645 pp->p_pagenum = pnum; 2646 2647 /* 2648 * Prefetch one more page_t into E$. To prevent future 2649 * mishaps with the sizeof(page_t) changing on us, we 2650 * catch this on debug kernels if we can't bring in the 2651 * entire hpage with 2 PREFETCH_BYTES reads. See 2652 * also, sun4u/cpu/cpu_module.c 2653 */ 2654 /*LINTED*/ 2655 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2656 prefetch_page_w((char *)pp); 2657 } 2658 2659 /* 2660 * kphysm_init() tackles the problem of initializing physical memory. 2661 * The old startup made some assumptions about the kernel living in 2662 * physically contiguous space which is no longer valid. 2663 */ 2664 static void 2665 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2666 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2667 { 2668 struct memlist *pmem; 2669 struct memseg *msp; 2670 pfn_t base; 2671 pgcnt_t num; 2672 pfn_t lastseg_pages_end = 0; 2673 pgcnt_t nelem_used = 0; 2674 2675 ASSERT(page_hash != NULL && page_hashsz != 0); 2676 2677 msp = memsegp; 2678 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2679 2680 /* 2681 * Build the memsegs entry 2682 */ 2683 num = btop(pmem->size); 2684 if (num > npages) 2685 num = npages; 2686 npages -= num; 2687 base = btop(pmem->address); 2688 2689 msp->pages = pp; 2690 msp->epages = pp + num; 2691 msp->pages_base = base; 2692 msp->pages_end = base + num; 2693 2694 if (kpm_enable) { 2695 pfn_t pbase_a; 2696 pfn_t pend_a; 2697 pfn_t prev_pend_a; 2698 pgcnt_t nelem; 2699 2700 msp->pagespa = va_to_pa(pp); 2701 msp->epagespa = va_to_pa(pp + num); 2702 pbase_a = kpmptop(ptokpmp(base)); 2703 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2704 nelem = ptokpmp(pend_a - pbase_a); 2705 msp->kpm_nkpmpgs = nelem; 2706 msp->kpm_pbase = pbase_a; 2707 if (lastseg_pages_end) { 2708 /* 2709 * Assume phys_avail is in ascending order 2710 * of physical addresses. 2711 */ 2712 ASSERT(base + num > lastseg_pages_end); 2713 prev_pend_a = kpmptop( 2714 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2715 2716 if (prev_pend_a > pbase_a) { 2717 /* 2718 * Overlap, more than one memseg may 2719 * point to the same kpm_page range. 2720 */ 2721 if (kpm_smallpages == 0) { 2722 msp->kpm_pages = 2723 (kpm_page_t *)kpm_pp - 1; 2724 kpm_pp = (uintptr_t) 2725 ((kpm_page_t *)kpm_pp 2726 + nelem - 1); 2727 } else { 2728 msp->kpm_spages = 2729 (kpm_spage_t *)kpm_pp - 1; 2730 kpm_pp = (uintptr_t) 2731 ((kpm_spage_t *)kpm_pp 2732 + nelem - 1); 2733 } 2734 nelem_used += nelem - 1; 2735 2736 } else { 2737 if (kpm_smallpages == 0) { 2738 msp->kpm_pages = 2739 (kpm_page_t *)kpm_pp; 2740 kpm_pp = (uintptr_t) 2741 ((kpm_page_t *)kpm_pp 2742 + nelem); 2743 } else { 2744 msp->kpm_spages = 2745 (kpm_spage_t *)kpm_pp; 2746 kpm_pp = (uintptr_t) 2747 ((kpm_spage_t *) 2748 kpm_pp + nelem); 2749 } 2750 nelem_used += nelem; 2751 } 2752 2753 } else { 2754 if (kpm_smallpages == 0) { 2755 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2756 kpm_pp = (uintptr_t) 2757 ((kpm_page_t *)kpm_pp + nelem); 2758 } else { 2759 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2760 kpm_pp = (uintptr_t) 2761 ((kpm_spage_t *)kpm_pp + nelem); 2762 } 2763 nelem_used = nelem; 2764 } 2765 2766 if (nelem_used > kpm_npages) 2767 panic("kphysm_init: kpm_pp overflow\n"); 2768 2769 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2770 lastseg_pages_end = msp->pages_end; 2771 } 2772 2773 memseg_list_add(msp); 2774 2775 /* 2776 * add_physmem() initializes the PSM part of the page 2777 * struct by calling the PSM back with add_physmem_cb(). 2778 * In addition it coalesces pages into larger pages as 2779 * it initializes them. 2780 */ 2781 add_physmem(pp, num, base); 2782 pp += num; 2783 msp++; 2784 } 2785 2786 build_pfn_hash(); 2787 } 2788 2789 /* 2790 * Kernel VM initialization. 2791 * Assumptions about kernel address space ordering: 2792 * (1) gap (user space) 2793 * (2) kernel text 2794 * (3) kernel data/bss 2795 * (4) gap 2796 * (5) kernel data structures 2797 * (6) gap 2798 * (7) debugger (optional) 2799 * (8) monitor 2800 * (9) gap (possibly null) 2801 * (10) dvma 2802 * (11) devices 2803 */ 2804 static void 2805 kvm_init(void) 2806 { 2807 /* 2808 * Put the kernel segments in kernel address space. 2809 */ 2810 rw_enter(&kas.a_lock, RW_WRITER); 2811 as_avlinit(&kas); 2812 2813 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2814 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2815 (void) segkmem_create(&ktextseg); 2816 2817 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2818 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2819 (void) segkmem_create(&ktexthole); 2820 2821 (void) seg_attach(&kas, (caddr_t)valloc_base, 2822 (size_t)(econtig32 - valloc_base), &kvalloc); 2823 (void) segkmem_create(&kvalloc); 2824 2825 if (kmem64_base) { 2826 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2827 (size_t)(kmem64_end - kmem64_base), &kmem64); 2828 (void) segkmem_create(&kmem64); 2829 } 2830 2831 /* 2832 * We're about to map out /boot. This is the beginning of the 2833 * system resource management transition. We can no longer 2834 * call into /boot for I/O or memory allocations. 2835 */ 2836 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2837 (void) segkmem_create(&kvseg); 2838 hblk_alloc_dynamic = 1; 2839 2840 /* 2841 * we need to preallocate pages for DR operations before enabling large 2842 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2843 */ 2844 memseg_remap_init(); 2845 2846 /* at this point we are ready to use large page heap */ 2847 segkmem_heap_lp_init(); 2848 2849 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2850 &kvseg32); 2851 (void) segkmem_create(&kvseg32); 2852 2853 /* 2854 * Create a segment for the debugger. 2855 */ 2856 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2857 (void) segkmem_create(&kdebugseg); 2858 2859 rw_exit(&kas.a_lock); 2860 } 2861 2862 char obp_tte_str[] = 2863 "h# %x constant MMU_PAGESHIFT " 2864 "h# %x constant TTE8K " 2865 "h# %x constant SFHME_SIZE " 2866 "h# %x constant SFHME_TTE " 2867 "h# %x constant HMEBLK_TAG " 2868 "h# %x constant HMEBLK_NEXT " 2869 "h# %x constant HMEBLK_MISC " 2870 "h# %x constant HMEBLK_HME1 " 2871 "h# %x constant NHMENTS " 2872 "h# %x constant HBLK_SZMASK " 2873 "h# %x constant HBLK_RANGE_SHIFT " 2874 "h# %x constant HMEBP_HBLK " 2875 "h# %x constant HMEBUCKET_SIZE " 2876 "h# %x constant HTAG_SFMMUPSZ " 2877 "h# %x constant HTAG_REHASHSZ " 2878 "h# %x constant mmu_hashcnt " 2879 "h# %p constant uhme_hash " 2880 "h# %p constant khme_hash " 2881 "h# %x constant UHMEHASH_SZ " 2882 "h# %x constant KHMEHASH_SZ " 2883 "h# %p constant KCONTEXT " 2884 "h# %p constant KHATID " 2885 "h# %x constant ASI_MEM " 2886 2887 ": PHYS-X@ ( phys -- data ) " 2888 " ASI_MEM spacex@ " 2889 "; " 2890 2891 ": PHYS-W@ ( phys -- data ) " 2892 " ASI_MEM spacew@ " 2893 "; " 2894 2895 ": PHYS-L@ ( phys -- data ) " 2896 " ASI_MEM spaceL@ " 2897 "; " 2898 2899 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2900 " 3 * MMU_PAGESHIFT + " 2901 "; " 2902 2903 ": TTE_IS_VALID ( ttep -- flag ) " 2904 " PHYS-X@ 0< " 2905 "; " 2906 2907 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2908 " dup TTE8K = if " 2909 " drop HBLK_RANGE_SHIFT " 2910 " else " 2911 " TTE_PAGE_SHIFT " 2912 " then " 2913 "; " 2914 2915 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2916 " tuck >> swap MMU_PAGESHIFT - << " 2917 "; " 2918 2919 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2920 " >> over xor swap ( hash sfmmup ) " 2921 " KHATID <> if ( hash ) " 2922 " UHMEHASH_SZ and ( bucket ) " 2923 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2924 " else ( hash ) " 2925 " KHMEHASH_SZ and ( bucket ) " 2926 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2927 " then ( hmebp ) " 2928 "; " 2929 2930 ": HME_HASH_TABLE_SEARCH " 2931 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2932 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2933 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2934 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2935 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2936 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2937 " else " 2938 " hmeblk_next + phys-x@ false " 2939 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2940 " then " 2941 " else " 2942 " hmeblk_next + phys-x@ false " 2943 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2944 " then " 2945 " else " 2946 " true " 2947 " then " 2948 " until r> drop " 2949 "; " 2950 2951 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2952 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2953 " HTAG_REHASHSZ << or nip ( hblktag ) " 2954 "; " 2955 2956 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2957 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2958 " TTE8K = if ( hmeblkp addr ) " 2959 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2960 " else ( hmeblkp addr ) " 2961 " drop 0 ( hmeblkp 0 ) " 2962 " then ( hmeblkp hme-index ) " 2963 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2964 " SFHME_TTE + ( ttep ) " 2965 "; " 2966 2967 ": unix-tte ( addr cnum -- false | tte-data true ) " 2968 " KCONTEXT = if ( addr ) " 2969 " KHATID ( addr khatid ) " 2970 " else ( addr ) " 2971 " drop false exit ( false ) " 2972 " then " 2973 " ( addr khatid ) " 2974 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2975 " 2dup swap i HME_HASH_SHIFT " 2976 "( addr sfmmup sfmmup addr hmeshift ) " 2977 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2978 " over i 4 pick " 2979 "( addr sfmmup hmebp sfmmup rehash addr ) " 2980 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2981 " HME_HASH_TABLE_SEARCH " 2982 "( addr sfmmup { null | hmeblkp } ) " 2983 " ?dup if ( addr sfmmup hmeblkp ) " 2984 " nip swap HBLK_TO_TTEP ( ttep ) " 2985 " dup TTE_IS_VALID if ( valid-ttep ) " 2986 " PHYS-X@ true ( tte-data true ) " 2987 " else ( invalid-tte ) " 2988 " drop false ( false ) " 2989 " then ( false | tte-data true ) " 2990 " unloop exit ( false | tte-data true ) " 2991 " then ( addr sfmmup ) " 2992 " loop ( addr sfmmup ) " 2993 " 2drop false ( false ) " 2994 "; " 2995 ; 2996 2997 void 2998 create_va_to_tte(void) 2999 { 3000 char *bp; 3001 extern int khmehash_num, uhmehash_num; 3002 extern struct hmehash_bucket *khme_hash, *uhme_hash; 3003 3004 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 3005 3006 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 3007 3008 /* 3009 * Teach obp how to parse our sw ttes. 3010 */ 3011 (void) sprintf(bp, obp_tte_str, 3012 MMU_PAGESHIFT, 3013 TTE8K, 3014 sizeof (struct sf_hment), 3015 OFFSET(struct sf_hment, hme_tte), 3016 OFFSET(struct hme_blk, hblk_tag), 3017 OFFSET(struct hme_blk, hblk_nextpa), 3018 OFFSET(struct hme_blk, hblk_misc), 3019 OFFSET(struct hme_blk, hblk_hme), 3020 NHMENTS, 3021 HBLK_SZMASK, 3022 HBLK_RANGE_SHIFT, 3023 OFFSET(struct hmehash_bucket, hmeh_nextpa), 3024 sizeof (struct hmehash_bucket), 3025 HTAG_SFMMUPSZ, 3026 HTAG_REHASHSZ, 3027 mmu_hashcnt, 3028 (caddr_t)va_to_pa((caddr_t)uhme_hash), 3029 (caddr_t)va_to_pa((caddr_t)khme_hash), 3030 UHMEHASH_SZ, 3031 KHMEHASH_SZ, 3032 KCONTEXT, 3033 KHATID, 3034 ASI_MEM); 3035 prom_interpret(bp, 0, 0, 0, 0, 0); 3036 3037 kobj_free(bp, MMU_PAGESIZE); 3038 } 3039 3040 void 3041 install_va_to_tte(void) 3042 { 3043 /* 3044 * advise prom that he can use unix-tte 3045 */ 3046 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3047 } 3048 3049 /* 3050 * Here we add "device-type=console" for /os-io node, for currently 3051 * our kernel console output only supports displaying text and 3052 * performing cursor-positioning operations (through kernel framebuffer 3053 * driver) and it doesn't support other functionalities required for a 3054 * standard "display" device as specified in 1275 spec. The main missing 3055 * interface defined by the 1275 spec is "draw-logo". 3056 * also see the comments above prom_stdout_is_framebuffer(). 3057 */ 3058 static char *create_node = 3059 "\" /\" find-device " 3060 "new-device " 3061 "\" os-io\" device-name " 3062 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3063 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3064 " 2>r swap 2 2r> ['] $callback catch if " 3065 " 2drop 3drop 0 " 3066 " then " 3067 "; " 3068 ": read ( adr,len -- #read ) " 3069 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3070 " ( retN ... ret1 N ) " 3071 " ?dup if " 3072 " swap >r 1- 0 ?do drop loop r> " 3073 " else " 3074 " -2 " 3075 " then " 3076 "; " 3077 ": write ( adr,len -- #written ) " 3078 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3079 " ( retN ... ret1 N ) " 3080 " ?dup if " 3081 " swap >r 1- 0 ?do drop loop r> " 3082 " else " 3083 " 0 " 3084 " then " 3085 "; " 3086 ": poll-tty ( -- ) ; " 3087 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3088 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3089 ": cb-give/take ( $method -- ) " 3090 " 0 -rot ['] $callback catch ?dup if " 3091 " >r 2drop 2drop r> throw " 3092 " else " 3093 " 0 ?do drop loop " 3094 " then " 3095 "; " 3096 ": give ( -- ) \" exit-input\" cb-give/take ; " 3097 ": take ( -- ) \" enter-input\" cb-give/take ; " 3098 ": open ( -- ok? ) true ; " 3099 ": close ( -- ) ; " 3100 "finish-device " 3101 "device-end "; 3102 3103 /* 3104 * Create the OBP input/output node (FCode serial driver). 3105 * It is needed for both USB console keyboard and for 3106 * the kernel terminal emulator. It is too early to check for a 3107 * kernel console compatible framebuffer now, so we create this 3108 * so that we're ready if we need to enable kernel terminal emulation. 3109 * 3110 * When the USB software takes over the input device at the time 3111 * consconfig runs, OBP's stdin is redirected to this node. 3112 * Whenever the FORTH user interface is used after this switch, 3113 * the node will call back into the kernel for console input. 3114 * If a serial device such as ttya or a UART with a Type 5 keyboard 3115 * attached is used, OBP takes over the serial device when the system 3116 * goes to the debugger after the system is booted. This sharing 3117 * of the relatively simple serial device is difficult but possible. 3118 * Sharing the USB host controller is impossible due its complexity. 3119 * 3120 * Similarly to USB keyboard input redirection, after consconfig_dacf 3121 * configures a kernel console framebuffer as the standard output 3122 * device, OBP's stdout is switched to to vector through the 3123 * /os-io node into the kernel terminal emulator. 3124 */ 3125 static void 3126 startup_create_io_node(void) 3127 { 3128 prom_interpret(create_node, 0, 0, 0, 0, 0); 3129 } 3130 3131 3132 static void 3133 do_prom_version_check(void) 3134 { 3135 int i; 3136 pnode_t node; 3137 char buf[64]; 3138 static char drev[] = "Down-rev firmware detected%s\n" 3139 "\tPlease upgrade to the following minimum version:\n" 3140 "\t\t%s\n"; 3141 3142 i = prom_version_check(buf, sizeof (buf), &node); 3143 3144 if (i == PROM_VER64_OK) 3145 return; 3146 3147 if (i == PROM_VER64_UPGRADE) { 3148 cmn_err(CE_WARN, drev, "", buf); 3149 3150 #ifdef DEBUG 3151 prom_enter_mon(); /* Type 'go' to continue */ 3152 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3153 return; 3154 #else 3155 halt(0); 3156 #endif 3157 } 3158 3159 /* 3160 * The other possibility is that this is a server running 3161 * good firmware, but down-rev firmware was detected on at 3162 * least one other cpu board. We just complain if we see 3163 * that. 3164 */ 3165 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3166 } 3167 3168 static void 3169 kpm_init() 3170 { 3171 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 3172 kpm_pgsz = 1ull << kpm_pgshft; 3173 kpm_pgoff = kpm_pgsz - 1; 3174 kpmp2pshft = kpm_pgshft - PAGESHIFT; 3175 kpmpnpgs = 1 << kpmp2pshft; 3176 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 3177 } 3178 3179 void 3180 kpm_npages_setup(int memblocks) 3181 { 3182 /* 3183 * npages can be scattered in a maximum of 'memblocks' 3184 */ 3185 kpm_npages = ptokpmpr(npages) + memblocks; 3186 } 3187 3188 /* 3189 * Must be defined in platform dependent code. 3190 */ 3191 extern caddr_t modtext; 3192 extern size_t modtext_sz; 3193 extern caddr_t moddata; 3194 3195 #define HEAPTEXT_ARENA(addr) \ 3196 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3197 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3198 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3199 3200 #define HEAPTEXT_OVERSIZED(addr) \ 3201 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3202 3203 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3204 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3205 kmutex_t texthole_lock; 3206 3207 char kern_bootargs[OBP_MAXPATHLEN]; 3208 3209 void 3210 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3211 { 3212 uintptr_t addr, limit; 3213 3214 addr = HEAPTEXT_BASE; 3215 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3216 3217 /* 3218 * Before we initialize the text_arena, we want to punch holes in the 3219 * underlying heaptext_arena. This guarantees that for any text 3220 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3221 */ 3222 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3223 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3224 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3225 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3226 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3227 } 3228 3229 /* 3230 * Allocate one page at the oversize to break up the text region 3231 * from the oversized region. 3232 */ 3233 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3234 (void *)limit, (void *)(limit + PAGESIZE), 3235 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3236 3237 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3238 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3239 heaptext_arena, 0, VM_SLEEP); 3240 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3241 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3242 } 3243 3244 caddr_t 3245 kobj_text_alloc(vmem_t *arena, size_t size) 3246 { 3247 caddr_t rval, better; 3248 3249 /* 3250 * First, try a sleeping allocation. 3251 */ 3252 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3253 3254 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3255 return (rval); 3256 3257 /* 3258 * We didn't get the area that we wanted. We're going to try to do an 3259 * allocation with explicit constraints. 3260 */ 3261 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3262 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3263 VM_NOSLEEP | VM_BESTFIT); 3264 3265 if (better != NULL) { 3266 /* 3267 * That worked. Free our first attempt and return. 3268 */ 3269 vmem_free(arena, rval, size); 3270 return (better); 3271 } 3272 3273 /* 3274 * That didn't work; we'll have to return our first attempt. 3275 */ 3276 return (rval); 3277 } 3278 3279 caddr_t 3280 kobj_texthole_alloc(caddr_t addr, size_t size) 3281 { 3282 int arena = HEAPTEXT_ARENA(addr); 3283 char c[30]; 3284 uintptr_t base; 3285 3286 if (HEAPTEXT_OVERSIZED(addr)) { 3287 /* 3288 * If this is an oversized allocation, there is no text hole 3289 * available for it; return NULL. 3290 */ 3291 return (NULL); 3292 } 3293 3294 mutex_enter(&texthole_lock); 3295 3296 if (texthole_arena[arena] == NULL) { 3297 ASSERT(texthole_source[arena] == NULL); 3298 3299 if (arena == 0) { 3300 texthole_source[0] = vmem_create("module_text_holesrc", 3301 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3302 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3303 0, VM_SLEEP); 3304 } else { 3305 base = HEAPTEXT_BASE + 3306 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3307 3308 (void) snprintf(c, sizeof (c), 3309 "heaptext_holesrc_%d", arena); 3310 3311 texthole_source[arena] = vmem_create(c, (void *)base, 3312 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3313 0, VM_SLEEP); 3314 } 3315 3316 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3317 3318 texthole_arena[arena] = vmem_create(c, NULL, 0, 3319 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3320 texthole_source[arena], 0, VM_SLEEP); 3321 } 3322 3323 mutex_exit(&texthole_lock); 3324 3325 ASSERT(texthole_arena[arena] != NULL); 3326 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3327 return (vmem_alloc(texthole_arena[arena], size, 3328 VM_BESTFIT | VM_NOSLEEP)); 3329 } 3330 3331 void 3332 kobj_texthole_free(caddr_t addr, size_t size) 3333 { 3334 int arena = HEAPTEXT_ARENA(addr); 3335 3336 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3337 ASSERT(texthole_arena[arena] != NULL); 3338 vmem_free(texthole_arena[arena], addr, size); 3339 } 3340