1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern void cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern void mach_dump_buffer_init(void); 74 extern void mach_descrip_init(void); 75 extern void mach_descrip_startup_fini(void); 76 extern void mach_memscrub(void); 77 extern void mach_fpras(void); 78 extern void mach_cpu_halt_idle(void); 79 extern void mach_hw_copy_limit(void); 80 extern void load_mach_drivers(void); 81 extern void load_tod_module(void); 82 #pragma weak load_tod_module 83 84 extern int ndata_alloc_mmfsa(struct memlist *ndata); 85 #pragma weak ndata_alloc_mmfsa 86 87 extern void cif_init(void); 88 #pragma weak cif_init 89 90 extern void parse_idprom(void); 91 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 92 extern void mem_config_init(void); 93 extern void memseg_remap_init(void); 94 95 extern void mach_kpm_init(void); 96 97 /* 98 * External Data: 99 */ 100 extern int vac_size; /* cache size in bytes */ 101 extern uint_t vac_mask; /* VAC alignment consistency mask */ 102 extern uint_t vac_colors; 103 104 /* 105 * Global Data Definitions: 106 */ 107 108 /* 109 * XXX - Don't port this to new architectures 110 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 111 * 'romp' has no use with a prom with an IEEE 1275 client interface. 112 * The driver doesn't use the value, but it depends on the symbol. 113 */ 114 void *romp; /* veritas driver won't load without romp 4154976 */ 115 /* 116 * Declare these as initialized data so we can patch them. 117 */ 118 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 119 pgcnt_t segkpsize = 120 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 121 uint_t segmap_percent = 12; /* Size of segmap segment */ 122 123 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 124 int vac_copyback = 1; 125 char *cache_mode = NULL; 126 int use_mix = 1; 127 int prom_debug = 0; 128 129 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 caddr_t sdata; /* beginning of data segment */ 151 152 caddr_t extra_etva; /* beginning of unused nucleus text */ 153 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 154 155 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 156 caddr_t nalloc_base; /* beginning of nucleus allocation */ 157 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 158 caddr_t valloc_base; /* beginning of kvalloc segment */ 159 160 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 161 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 162 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 163 int kmem64_alignsize; /* page size for mem segment in 64-bit space */ 164 int kmem64_szc; /* page size code */ 165 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 166 167 uintptr_t shm_alignment; /* VAC address consistency modulus */ 168 struct memlist *phys_install; /* Total installed physical memory */ 169 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 170 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 171 struct memlist ndata; /* memlist of nucleus allocatable memory */ 172 int memexp_flag; /* memory expansion card flag */ 173 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 174 pgcnt_t obp_pages; /* Physical pages used by OBP */ 175 176 /* 177 * VM data structures 178 */ 179 long page_hashsz; /* Size of page hash table (power of two) */ 180 struct page *pp_base; /* Base of system page struct array */ 181 size_t pp_sz; /* Size in bytes of page struct array */ 182 struct page **page_hash; /* Page hash table */ 183 struct seg ktextseg; /* Segment used for kernel executable image */ 184 struct seg kvalloc; /* Segment used for "valloc" mapping */ 185 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 186 struct seg ktexthole; /* Segment used for nucleus text hole */ 187 struct seg kmapseg; /* Segment used for generic kernel mappings */ 188 struct seg kpmseg; /* Segment used for physical mapping */ 189 struct seg kdebugseg; /* Segment used for the kernel debugger */ 190 191 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 192 size_t kpm_pp_sz; /* Size of system kpm_page array */ 193 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 194 195 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 196 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 197 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 198 199 int segzio_fromheap = 0; /* zio allocations occur from heap */ 200 caddr_t segzio_base; /* Base address of segzio */ 201 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 202 203 /* 204 * debugger pages (if allocated) 205 */ 206 struct vnode kdebugvp; 207 208 /* 209 * VA range available to the debugger 210 */ 211 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 212 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 213 214 /* 215 * Segment for relocated kernel structures in 64-bit large RAM kernels 216 */ 217 struct seg kmem64; 218 219 struct memseg *memseg_base; 220 size_t memseg_sz; /* Used to translate a va to page */ 221 struct vnode unused_pages_vp; 222 223 /* 224 * VM data structures allocated early during boot. 225 */ 226 size_t pagehash_sz; 227 uint64_t memlist_sz; 228 229 char tbr_wr_addr_inited = 0; 230 231 232 /* 233 * Static Routines: 234 */ 235 static void memlist_add(uint64_t, uint64_t, struct memlist **, 236 struct memlist **); 237 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 238 pgcnt_t); 239 static void kvm_init(void); 240 241 static void startup_init(void); 242 static void startup_memlist(void); 243 static void startup_modules(void); 244 static void startup_bop_gone(void); 245 static void startup_vm(void); 246 static void startup_end(void); 247 static void setup_cage_params(void); 248 static void startup_create_io_node(void); 249 250 static pgcnt_t npages; 251 static struct memlist *memlist; 252 void *memlist_end; 253 254 static pgcnt_t bop_alloc_pages; 255 static caddr_t hblk_base; 256 uint_t hblk_alloc_dynamic = 0; 257 uint_t hblk1_min = H1MIN; 258 259 260 /* 261 * Hooks for unsupported platforms and down-rev firmware 262 */ 263 int iam_positron(void); 264 #pragma weak iam_positron 265 static void do_prom_version_check(void); 266 static void kpm_init(void); 267 static void kpm_npages_setup(int); 268 static void kpm_memseg_init(void); 269 270 /* 271 * After receiving a thermal interrupt, this is the number of seconds 272 * to delay before shutting off the system, assuming 273 * shutdown fails. Use /etc/system to change the delay if this isn't 274 * large enough. 275 */ 276 int thermal_powerdown_delay = 1200; 277 278 /* 279 * Used to hold off page relocations into the cage until OBP has completed 280 * its boot-time handoff of its resources to the kernel. 281 */ 282 int page_relocate_ready = 0; 283 284 /* 285 * Enable some debugging messages concerning memory usage... 286 */ 287 #ifdef DEBUGGING_MEM 288 static int debugging_mem; 289 static void 290 printmemlist(char *title, struct memlist *list) 291 { 292 if (!debugging_mem) 293 return; 294 295 printf("%s\n", title); 296 297 while (list) { 298 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 299 (uint32_t)(list->address >> 32), (uint32_t)list->address, 300 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 301 list = list->next; 302 } 303 } 304 305 void 306 printmemseg(struct memseg *memseg) 307 { 308 if (!debugging_mem) 309 return; 310 311 printf("memseg\n"); 312 313 while (memseg) { 314 prom_printf("\tpage = 0x%p, epage = 0x%p, " 315 "pfn = 0x%x, epfn = 0x%x\n", 316 memseg->pages, memseg->epages, 317 memseg->pages_base, memseg->pages_end); 318 memseg = memseg->next; 319 } 320 } 321 322 #define debug_pause(str) halt((str)) 323 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 324 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 325 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 326 #define MPRINTF3(str, a, b, c) \ 327 if (debugging_mem) prom_printf((str), (a), (b), (c)) 328 #else /* DEBUGGING_MEM */ 329 #define MPRINTF(str) 330 #define MPRINTF1(str, a) 331 #define MPRINTF2(str, a, b) 332 #define MPRINTF3(str, a, b, c) 333 #endif /* DEBUGGING_MEM */ 334 335 /* Simple message to indicate that the bootops pointer has been zeroed */ 336 #ifdef DEBUG 337 static int bootops_gone_on = 0; 338 #define BOOTOPS_GONE() \ 339 if (bootops_gone_on) \ 340 prom_printf("The bootops vec is zeroed now!\n"); 341 #else 342 #define BOOTOPS_GONE() 343 #endif /* DEBUG */ 344 345 /* 346 * Monitor pages may not be where this says they are. 347 * and the debugger may not be there either. 348 * 349 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 350 * 351 * Physical memory layout 352 * (not necessarily contiguous) 353 * (THIS IS SOMEWHAT WRONG) 354 * /-----------------------\ 355 * | monitor pages | 356 * availmem -|-----------------------| 357 * | | 358 * | page pool | 359 * | | 360 * |-----------------------| 361 * | configured tables | 362 * | buffers | 363 * firstaddr -|-----------------------| 364 * | hat data structures | 365 * |-----------------------| 366 * | kernel data, bss | 367 * |-----------------------| 368 * | interrupt stack | 369 * |-----------------------| 370 * | kernel text (RO) | 371 * |-----------------------| 372 * | trap table (4k) | 373 * |-----------------------| 374 * page 1 | panicbuf | 375 * |-----------------------| 376 * page 0 | reclaimed | 377 * |_______________________| 378 * 379 * 380 * 381 * Kernel's Virtual Memory Layout. 382 * /-----------------------\ 383 * 0xFFFFFFFF.FFFFFFFF -| |- 384 * | OBP's virtual page | 385 * | tables | 386 * 0xFFFFFFFC.00000000 -|-----------------------|- 387 * : : 388 * : : 389 * -|-----------------------|- 390 * | segzio | (base and size vary) 391 * 0xFFFFFE00.00000000 -|-----------------------|- 392 * | | Ultrasparc I/II support 393 * | segkpm segment | up to 2TB of physical 394 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 395 * | | 396 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 397 * : : 398 * : : 399 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 400 * | | ^ 401 * | UltraSPARC I/II call | | 402 * | bug requires an extra | | 403 * | 4 GB of space between | | 404 * | hole and used RAM | | 405 * | | | 406 * 0xFFFFF800.00000000 -|-----------------------|- | 407 * | | | 408 * | Virtual Address Hole | UltraSPARC 409 * | on UltraSPARC I/II | I/II * ONLY * 410 * | | | 411 * 0x00000800.00000000 -|-----------------------|- | 412 * | | | 413 * | UltraSPARC I/II call | | 414 * | bug requires an extra | | 415 * | 4 GB of space between | | 416 * | hole and used RAM | | 417 * | | v 418 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 419 * : : ^ 420 * : : | 421 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 422 * | overmapped area | alignend_end | 423 * | (kmem64_alignsize | | 424 * | boundary) | | 425 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 426 * | | | 427 * | 64-bit kernel ONLY | | 428 * | | | 429 * | kmem64 segment | | 430 * | | | 431 * | (Relocated extra HME | Approximately 432 * | block allocations, | 1 TB of virtual 433 * | memnode freelists, | address space 434 * | HME hash buckets, | | 435 * | mml_table, kpmp_table,| | 436 * | page_t array and | | 437 * | hashblock pool to | | 438 * | avoid hard-coded | | 439 * | 32-bit vaddr | | 440 * | limitations) | | 441 * | | v 442 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 443 * | | 444 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 445 * | | 446 * 0x00000300.00000000 -|-----------------------|- SYSBASE 447 * : : 448 * : : 449 * -|-----------------------|- 450 * | | 451 * | segmap segment | SEGMAPSIZE (1/8th physmem, 452 * | | 256G MAX) 453 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 454 * : : 455 * : : 456 * -|-----------------------|- 457 * | | 458 * | segkp | SEGKPSIZE (2GB) 459 * | | 460 * | | 461 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 462 * | | 463 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 464 * | | (SEGKPBASE - 0x400000) 465 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 466 * | | (MEMSCRUBBASE - NCARGS) 467 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 468 * | | (ARGSBASE - PPMAPSIZE) 469 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 470 * | | 471 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 472 * | | 473 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 474 * : : 475 * : : 476 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 477 * | | 478 * | OBP | 479 * | | 480 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 481 * | kmdb | 482 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 483 * : : 484 * : : 485 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 486 * | | 487 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 488 * | | ~64MB) 489 * 0x00000000.78002000 -|-----------------------| 490 * | panicbuf | 491 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 492 * : : 493 * : : 494 * | | 495 * |-----------------------|- econtig32 496 * | vm structures | 497 * 0x00000000.01C00000 |-----------------------|- nalloc_end 498 * | TSBs | 499 * |-----------------------|- end/nalloc_base 500 * | kernel data & bss | 501 * 0x00000000.01800000 -|-----------------------| 502 * : nucleus text hole : 503 * 0x00000000.01400000 -|-----------------------| 504 * : : 505 * |-----------------------| 506 * | module text | 507 * |-----------------------|- e_text/modtext 508 * | kernel text | 509 * |-----------------------| 510 * | trap table (48k) | 511 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 512 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 513 * |-----------------------| 514 * | | 515 * | invalid | 516 * | | 517 * 0x00000000.00000000 _|_______________________| 518 * 519 * 520 * 521 * 32-bit User Virtual Memory Layout. 522 * /-----------------------\ 523 * | | 524 * | invalid | 525 * | | 526 * 0xFFC00000 -|-----------------------|- USERLIMIT 527 * | user stack | 528 * : : 529 * : : 530 * : : 531 * | user data | 532 * -|-----------------------|- 533 * | user text | 534 * 0x00002000 -|-----------------------|- 535 * | invalid | 536 * 0x00000000 _|_______________________| 537 * 538 * 539 * 540 * 64-bit User Virtual Memory Layout. 541 * /-----------------------\ 542 * | | 543 * | invalid | 544 * | | 545 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 546 * | user stack | 547 * : : 548 * : : 549 * : : 550 * | user data | 551 * -|-----------------------|- 552 * | user text | 553 * 0x00000000.00100000 -|-----------------------|- 554 * | invalid | 555 * 0x00000000.00000000 _|_______________________| 556 */ 557 558 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 559 extern uint64_t ecache_flush_address(void); 560 561 #pragma weak load_platform_modules 562 #pragma weak plat_startup_memlist 563 #pragma weak ecache_init_scrub_flush_area 564 #pragma weak ecache_flush_address 565 566 567 /* 568 * By default the DR Cage is enabled for maximum OS 569 * MPSS performance. Users needing to disable the cage mechanism 570 * can set this variable to zero via /etc/system. 571 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 572 * will result in loss of DR functionality. 573 * Platforms wishing to disable kernel Cage by default 574 * should do so in their set_platform_defaults() routine. 575 */ 576 int kernel_cage_enable = 1; 577 578 static void 579 setup_cage_params(void) 580 { 581 void (*func)(void); 582 583 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 584 if (func != NULL) { 585 (*func)(); 586 return; 587 } 588 589 if (kernel_cage_enable == 0) { 590 return; 591 } 592 kcage_range_lock(); 593 if (kcage_range_init(phys_avail, 1) == 0) { 594 kcage_init(total_pages / 256); 595 } 596 kcage_range_unlock(); 597 598 if (kcage_on) { 599 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 600 } else { 601 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 602 } 603 604 } 605 606 /* 607 * Machine-dependent startup code 608 */ 609 void 610 startup(void) 611 { 612 startup_init(); 613 if (&startup_platform) 614 startup_platform(); 615 startup_memlist(); 616 startup_modules(); 617 setup_cage_params(); 618 startup_bop_gone(); 619 startup_vm(); 620 startup_end(); 621 } 622 623 struct regs sync_reg_buf; 624 uint64_t sync_tt; 625 626 void 627 sync_handler(void) 628 { 629 struct trap_info ti; 630 int i; 631 632 /* 633 * Prevent trying to talk to the other CPUs since they are 634 * sitting in the prom and won't reply. 635 */ 636 for (i = 0; i < NCPU; i++) { 637 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 638 cpu[i]->cpu_flags &= ~CPU_READY; 639 cpu[i]->cpu_flags |= CPU_QUIESCED; 640 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 641 } 642 } 643 644 /* 645 * We've managed to get here without going through the 646 * normal panic code path. Try and save some useful 647 * information. 648 */ 649 if (!panicstr && (curthread->t_panic_trap == NULL)) { 650 ti.trap_type = sync_tt; 651 ti.trap_regs = &sync_reg_buf; 652 ti.trap_addr = NULL; 653 ti.trap_mmu_fsr = 0x0; 654 655 curthread->t_panic_trap = &ti; 656 } 657 658 /* 659 * If we're re-entering the panic path, update the signature 660 * block so that the SC knows we're in the second part of panic. 661 */ 662 if (panicstr) 663 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 664 665 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 666 panic("sync initiated"); 667 } 668 669 670 static void 671 startup_init(void) 672 { 673 /* 674 * We want to save the registers while we're still in OBP 675 * so that we know they haven't been fiddled with since. 676 * (In principle, OBP can't change them just because it 677 * makes a callback, but we'd rather not depend on that 678 * behavior.) 679 */ 680 char sync_str[] = 681 "warning @ warning off : sync " 682 "%%tl-c %%tstate h# %p x! " 683 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 684 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 685 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 686 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 687 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 688 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 689 "%%y h# %p l! %%tl-c %%tt h# %p x! " 690 "sync ; warning !"; 691 692 /* 693 * 20 == num of %p substrings 694 * 16 == max num of chars %p will expand to. 695 */ 696 char bp[sizeof (sync_str) + 16 * 20]; 697 698 (void) check_boot_version(BOP_GETVERSION(bootops)); 699 700 /* 701 * Initialize ptl1 stack for the 1st CPU. 702 */ 703 ptl1_init_cpu(&cpu0); 704 705 /* 706 * Initialize the address map for cache consistent mappings 707 * to random pages; must be done after vac_size is set. 708 */ 709 ppmapinit(); 710 711 /* 712 * Initialize the PROM callback handler. 713 */ 714 init_vx_handler(); 715 716 /* 717 * have prom call sync_callback() to handle the sync and 718 * save some useful information which will be stored in the 719 * core file later. 720 */ 721 (void) sprintf((char *)bp, sync_str, 722 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 723 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 724 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 725 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 726 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 727 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 728 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 729 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 730 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 731 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 732 prom_interpret(bp, 0, 0, 0, 0, 0); 733 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 734 } 735 736 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 737 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 738 739 #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 740 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 741 (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 742 743 #if !defined(C_OBP) 744 /* 745 * Install a temporary tte handler in OBP for kmem64 area. 746 * 747 * We map kmem64 area with large pages before the trap table is taken 748 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 749 * the same area. Duplicate tlb entries with different page sizes 750 * cause unpredicatble behavior. To avoid this, we don't create 751 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 752 * OBP). Instead, we manage translations with a temporary va>tte-data 753 * handler (kmem64-tte). This handler is replaced by unix-tte when 754 * the trap table is taken over. 755 * 756 * The temporary handler knows the physical address of the kmem64 757 * area. It uses the prom's pgmap@ Forth word for other addresses. 758 * 759 * We have to use BOP_ALLOC() method for C-OBP platforms because 760 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 761 * sun4u platforms. On sun4u we flush tlb after trap table is taken 762 * over if we use large pages for kernel heap and kmem64. Since sun4u 763 * prom (unlike sun4v) calls va>tte-data first for client address 764 * translation prom's ttes for kmem64 can't get into TLB even if we 765 * later switch to prom's trap table again. C-OBP uses 4M pages for 766 * client mappings when possible so on all platforms we get the 767 * benefit from large mappings for kmem64 area immediately during 768 * boot. 769 * 770 * pseudo code: 771 * if (context != 0) { 772 * return false 773 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 774 * tte = tte_template + 775 * (((miss_va & pagemask) - kmem64_base)); 776 * return tte, true 777 * } else { 778 * return pgmap@ result 779 * } 780 */ 781 char kmem64_obp_str[] = 782 "h# %lx constant kmem64_base " 783 "h# %lx constant kmem64_end " 784 "h# %lx constant kmem64_pagemask " 785 "h# %lx constant kmem64_template " 786 787 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 788 " if ( addr ) " 789 " drop false exit then ( false ) " 790 " dup kmem64_base kmem64_end within if ( addr ) " 791 " kmem64_pagemask and ( addr' ) " 792 " kmem64_base - ( addr' ) " 793 " kmem64_template + ( tte ) " 794 " true ( tte true ) " 795 " else ( addr ) " 796 " pgmap@ ( tte ) " 797 " dup 0< if true else drop false then ( tte true | false ) " 798 " then ( tte true | false ) " 799 "; " 800 801 "' kmem64-tte is va>tte-data " 802 ; 803 804 void 805 install_kmem64_tte() 806 { 807 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 808 tte_t tte; 809 810 PRM_DEBUG(kmem64_pabase); 811 PRM_DEBUG(kmem64_szc); 812 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 813 PROC_DATA | HAT_NOSYNC, kmem64_szc); 814 PRM_DEBUG(tte.ll); 815 (void) sprintf(b, kmem64_obp_str, 816 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 817 ASSERT(strlen(b) < sizeof (b)); 818 prom_interpret(b, 0, 0, 0, 0, 0); 819 } 820 #endif /* !C_OBP */ 821 822 /* 823 * As OBP takes up some RAM when the system boots, pages will already be "lost" 824 * to the system and reflected in npages by the time we see it. 825 * 826 * We only want to allocate kernel structures in the 64-bit virtual address 827 * space on systems with enough RAM to make the overhead of keeping track of 828 * an extra kernel memory segment worthwhile. 829 * 830 * Since OBP has already performed its memory allocations by this point, if we 831 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 832 * memory in the 64-bit virtual address space; otherwise keep allocations 833 * contiguous with we've mapped so far in the 32-bit virtual address space. 834 */ 835 #define MINMOVE_RAM_MB ((size_t)1900) 836 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 837 838 pgcnt_t tune_npages = (pgcnt_t) 839 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 840 841 #pragma weak page_set_colorequiv_arr_cpu 842 extern void page_set_colorequiv_arr_cpu(void); 843 844 static void 845 startup_memlist(void) 846 { 847 size_t alloc_sz; 848 size_t ctrs_sz; 849 caddr_t alloc_base; 850 caddr_t ctrs_base, ctrs_end; 851 caddr_t memspace; 852 caddr_t va; 853 int memblocks = 0; 854 struct memlist *cur; 855 size_t syslimit = (size_t)SYSLIMIT; 856 size_t sysbase = (size_t)SYSBASE; 857 int alloc_alignsize = ecache_alignsize; 858 int i; 859 extern void page_coloring_init(void); 860 extern void page_set_colorequiv_arr(void); 861 862 /* 863 * Initialize enough of the system to allow kmem_alloc to work by 864 * calling boot to allocate its memory until the time that 865 * kvm_init is completed. The page structs are allocated after 866 * rounding up end to the nearest page boundary; the memsegs are 867 * initialized and the space they use comes from the kernel heap. 868 * With appropriate initialization, they can be reallocated later 869 * to a size appropriate for the machine's configuration. 870 * 871 * At this point, memory is allocated for things that will never 872 * need to be freed, this used to be "valloced". This allows a 873 * savings as the pages don't need page structures to describe 874 * them because them will not be managed by the vm system. 875 */ 876 877 /* 878 * We're loaded by boot with the following configuration (as 879 * specified in the sun4u/conf/Mapfile): 880 * 881 * text: 4 MB chunk aligned on a 4MB boundary 882 * data & bss: 4 MB chunk aligned on a 4MB boundary 883 * 884 * These two chunks will eventually be mapped by 2 locked 4MB 885 * ttes and will represent the nucleus of the kernel. This gives 886 * us some free space that is already allocated, some or all of 887 * which is made available to kernel module text. 888 * 889 * The free space in the data-bss chunk is used for nucleus 890 * allocatable data structures and we reserve it using the 891 * nalloc_base and nalloc_end variables. This space is currently 892 * being used for hat data structures required for tlb miss 893 * handling operations. We align nalloc_base to a l2 cache 894 * linesize because this is the line size the hardware uses to 895 * maintain cache coherency. 896 * 256K is carved out for module data. 897 */ 898 899 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 900 moddata = nalloc_base; 901 e_moddata = nalloc_base + MODDATA; 902 nalloc_base = e_moddata; 903 904 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 905 valloc_base = nalloc_base; 906 907 /* 908 * Calculate the start of the data segment. 909 */ 910 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 911 912 PRM_DEBUG(moddata); 913 PRM_DEBUG(nalloc_base); 914 PRM_DEBUG(nalloc_end); 915 PRM_DEBUG(sdata); 916 917 /* 918 * Remember any slop after e_text so we can give it to the modules. 919 */ 920 PRM_DEBUG(e_text); 921 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 922 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 923 prom_panic("nucleus text overflow"); 924 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 925 modtext; 926 PRM_DEBUG(modtext); 927 PRM_DEBUG(modtext_sz); 928 929 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 930 &boot_physavail, &boot_physavail_len, 931 &boot_virtavail, &boot_virtavail_len); 932 /* 933 * Remember what the physically available highest page is 934 * so that dumpsys works properly, and find out how much 935 * memory is installed. 936 */ 937 installed_top_size_memlist_array(boot_physinstalled, 938 boot_physinstalled_len, &physmax, &physinstalled); 939 PRM_DEBUG(physinstalled); 940 PRM_DEBUG(physmax); 941 942 /* Fill out memory nodes config structure */ 943 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 944 945 /* 946 * Get the list of physically available memory to size 947 * the number of page structures needed. 948 */ 949 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 950 /* 951 * This first snap shot of npages can represent the pages used 952 * by OBP's text and data approximately. This is used in the 953 * the calculation of the kernel size 954 */ 955 obp_pages = physinstalled - npages; 956 957 958 /* 959 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 960 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 961 * 2MB) and any excess pages are put on physavail. The assumption is 962 * that small-memory systems will need more pages more than they'll 963 * need efficiently-mapped module texts. 964 */ 965 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 966 modtext_sz > MODTEXT_SM_CAP) { 967 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 968 modtext_sz = MODTEXT_SM_CAP; 969 extra_etva = modtext + modtext_sz; 970 } 971 972 PRM_DEBUG(extra_etpg); 973 PRM_DEBUG(modtext_sz); 974 PRM_DEBUG(extra_etva); 975 976 /* 977 * Account for any pages after e_text and e_data. 978 */ 979 npages += extra_etpg; 980 npages += mmu_btopr(nalloc_end - nalloc_base); 981 PRM_DEBUG(npages); 982 983 /* 984 * npages is the maximum of available physical memory possible. 985 * (ie. it will never be more than this) 986 */ 987 988 /* 989 * initialize the nucleus memory allocator. 990 */ 991 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 992 993 /* 994 * Allocate mmu fault status area from the nucleus data area. 995 */ 996 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 997 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 998 999 /* 1000 * Allocate kernel TSBs from the nucleus data area. 1001 */ 1002 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1003 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1004 1005 /* 1006 * Allocate dmv dispatch table from the nucleus data area. 1007 */ 1008 if (ndata_alloc_dmv(&ndata) != 0) 1009 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1010 1011 1012 page_coloring_init(); 1013 1014 /* 1015 * Allocate page_freelists bin headers for memnode 0 from the 1016 * nucleus data area. 1017 */ 1018 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 1019 cmn_err(CE_PANIC, 1020 "no more nucleus memory after page free lists alloc"); 1021 1022 if (kpm_enable) { 1023 kpm_init(); 1024 /* 1025 * kpm page space -- Update kpm_npages and make the 1026 * same assumption about fragmenting as it is done 1027 * for memseg_sz. 1028 */ 1029 kpm_npages_setup(memblocks + 4); 1030 } 1031 1032 /* 1033 * Allocate hat related structs from the nucleus data area. 1034 */ 1035 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 1036 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1037 1038 /* 1039 * We want to do the BOP_ALLOCs before the real allocation of page 1040 * structs in order to not have to allocate page structs for this 1041 * memory. We need to calculate a virtual address because we want 1042 * the page structs to come before other allocations in virtual address 1043 * space. This is so some (if not all) of page structs can actually 1044 * live in the nucleus. 1045 */ 1046 1047 /* 1048 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1049 * 1050 * There are comments all over the SFMMU code warning of dire 1051 * consequences if the TSBs are moved out of 32-bit space. This 1052 * is largely because the asm code uses "sethi %hi(addr)"-type 1053 * instructions which will not provide the expected result if the 1054 * address is a 64-bit one. 1055 * 1056 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1057 */ 1058 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1059 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1060 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1061 PRM_DEBUG(alloc_base); 1062 1063 /* 1064 * Allocate IOMMU TSB array. We do this here so that the physical 1065 * memory gets deducted from the PROM's physical memory list. 1066 */ 1067 alloc_base = iommu_tsb_init(alloc_base); 1068 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1069 ecache_alignsize); 1070 PRM_DEBUG(alloc_base); 1071 1072 /* 1073 * Platforms like Starcat and OPL need special structures assigned in 1074 * 32-bit virtual address space because their probing routines execute 1075 * FCode, and FCode can't handle 64-bit virtual addresses... 1076 */ 1077 if (&plat_startup_memlist) { 1078 alloc_base = plat_startup_memlist(alloc_base); 1079 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1080 ecache_alignsize); 1081 PRM_DEBUG(alloc_base); 1082 } 1083 1084 /* 1085 * Save off where the contiguous allocations to date have ended 1086 * in econtig32. 1087 */ 1088 econtig32 = alloc_base; 1089 PRM_DEBUG(econtig32); 1090 1091 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1092 cmn_err(CE_PANIC, "econtig32 too big"); 1093 1094 /* 1095 * To avoid memory allocation collisions in the 32-bit virtual address 1096 * space, make allocations from this point forward in 64-bit virtual 1097 * address space starting at syslimit and working up. 1098 * 1099 * All this is needed because on large memory systems, the default 1100 * Solaris allocations will collide with SYSBASE32, which is hard 1101 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1102 * kernels, move the allocations to a location in the 64-bit virtual 1103 * address space space, allowing those structures to grow without 1104 * worry. 1105 * 1106 * On current CPUs we'll run out of physical memory address bits before 1107 * we need to worry about the allocations running into anything else in 1108 * VM or the virtual address holes on US-I and II, as there's currently 1109 * about 1 TB of addressable space before the US-I/II VA hole. 1110 */ 1111 kmem64_base = (caddr_t)syslimit; 1112 PRM_DEBUG(kmem64_base); 1113 1114 /* 1115 * Allocate addresses, but not physical memory. None of these locations 1116 * can be touched until physical memory is allocated below. 1117 */ 1118 alloc_base = kmem64_base; 1119 1120 /* 1121 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1122 * them here. 1123 */ 1124 if (khme_hash == NULL || uhme_hash == NULL) { 1125 /* 1126 * alloc_hme_buckets() will align alloc_base properly before 1127 * assigning the hash buckets, so we don't need to do it 1128 * before the call... 1129 */ 1130 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1131 1132 PRM_DEBUG(alloc_base); 1133 PRM_DEBUG(khme_hash); 1134 PRM_DEBUG(uhme_hash); 1135 } 1136 1137 /* 1138 * Allocate the remaining page freelists. NUMA systems can 1139 * have lots of page freelists, one per node, which quickly 1140 * outgrow the amount of nucleus memory available. 1141 */ 1142 if (max_mem_nodes > 1) { 1143 int mnode; 1144 1145 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1146 alloc_base = alloc_page_freelists(mnode, alloc_base, 1147 ecache_alignsize); 1148 } 1149 PRM_DEBUG(alloc_base); 1150 } 1151 1152 if (!mml_table) { 1153 size_t mmltable_sz; 1154 1155 /* 1156 * We need to allocate the mml_table here because there 1157 * was not enough space within the nucleus. 1158 */ 1159 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1160 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1161 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1162 alloc_alignsize); 1163 mml_table = (kmutex_t *)alloc_base; 1164 alloc_base += alloc_sz; 1165 PRM_DEBUG(mml_table); 1166 PRM_DEBUG(alloc_base); 1167 } 1168 1169 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1170 size_t kpmptable_sz; 1171 caddr_t table; 1172 1173 /* 1174 * We need to allocate either kpmp_table or kpmp_stable here 1175 * because there was not enough space within the nucleus. 1176 */ 1177 kpmptable_sz = (kpm_smallpages == 0) ? 1178 sizeof (kpm_hlk_t) * kpmp_table_sz : 1179 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1180 1181 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1182 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1183 alloc_alignsize); 1184 1185 table = alloc_base; 1186 1187 if (kpm_smallpages == 0) { 1188 kpmp_table = (kpm_hlk_t *)table; 1189 PRM_DEBUG(kpmp_table); 1190 } else { 1191 kpmp_stable = (kpm_shlk_t *)table; 1192 PRM_DEBUG(kpmp_stable); 1193 } 1194 1195 alloc_base += alloc_sz; 1196 PRM_DEBUG(alloc_base); 1197 } 1198 1199 if (&ecache_init_scrub_flush_area) { 1200 /* 1201 * Pass alloc_base directly, as the routine itself is 1202 * responsible for any special alignment requirements... 1203 */ 1204 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1205 PRM_DEBUG(alloc_base); 1206 } 1207 1208 /* 1209 * Take the most current snapshot we can by calling mem-update. 1210 */ 1211 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1212 &boot_physavail, &boot_physavail_len, 1213 &boot_virtavail, &boot_virtavail_len); 1214 1215 /* 1216 * Reset npages and memblocks based on boot_physavail list. 1217 */ 1218 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1219 PRM_DEBUG(npages); 1220 1221 /* 1222 * Account for extra memory after e_text. 1223 */ 1224 npages += extra_etpg; 1225 1226 /* 1227 * Calculate the largest free memory chunk in the nucleus data area. 1228 * We need to figure out if page structs can fit in there or not. 1229 * We also make sure enough page structs get created for any physical 1230 * memory we might be returning to the system. 1231 */ 1232 ndata_remain_sz = ndata_maxsize(&ndata); 1233 PRM_DEBUG(ndata_remain_sz); 1234 1235 pp_sz = sizeof (struct page) * npages; 1236 1237 /* 1238 * Here's a nice bit of code based on somewhat recursive logic: 1239 * 1240 * If the page array would fit within the nucleus, we want to 1241 * add npages to cover any extra memory we may be returning back 1242 * to the system. 1243 * 1244 * HOWEVER, the page array is sized by calculating the size of 1245 * (struct page * npages), as are the pagehash table, ctrs and 1246 * memseg_list, so the very act of performing the calculation below may 1247 * in fact make the array large enough that it no longer fits in the 1248 * nucleus, meaning there would now be a much larger area of the 1249 * nucleus free that should really be added to npages, which would 1250 * make the page array that much larger, and so on. 1251 * 1252 * This also ignores the memory possibly used in the nucleus for the 1253 * the page hash, ctrs and memseg list and the fact that whether they 1254 * fit there or not varies with the npages calculation below, but we 1255 * don't even factor them into the equation at this point; perhaps we 1256 * should or perhaps we should just take the approach that the few 1257 * extra pages we could add via this calculation REALLY aren't worth 1258 * the hassle... 1259 */ 1260 if (ndata_remain_sz > pp_sz) { 1261 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1262 1263 npages += mmu_btop(spare); 1264 1265 pp_sz = npages * sizeof (struct page); 1266 1267 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1268 } 1269 1270 /* 1271 * If physmem is patched to be non-zero, use it instead of 1272 * the monitor value unless physmem is larger than the total 1273 * amount of memory on hand. 1274 */ 1275 if (physmem == 0 || physmem > npages) 1276 physmem = npages; 1277 1278 /* 1279 * If pp_base is NULL that means the routines above have determined 1280 * the page array will not fit in the nucleus; we'll have to 1281 * BOP_ALLOC() ourselves some space for them. 1282 */ 1283 if (pp_base == NULL) { 1284 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1285 alloc_alignsize); 1286 alloc_sz = roundup(pp_sz, alloc_alignsize); 1287 1288 pp_base = (struct page *)alloc_base; 1289 1290 alloc_base += alloc_sz; 1291 } 1292 1293 /* 1294 * The page structure hash table size is a power of 2 1295 * such that the average hash chain length is PAGE_HASHAVELEN. 1296 */ 1297 page_hashsz = npages / PAGE_HASHAVELEN; 1298 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1299 pagehash_sz = sizeof (struct page *) * page_hashsz; 1300 1301 /* 1302 * We want to TRY to fit the page structure hash table, 1303 * the page size free list counters, the memseg list and 1304 * and the kpm page space in the nucleus if possible. 1305 * 1306 * alloc_sz counts how much memory needs to be allocated by 1307 * BOP_ALLOC(). 1308 */ 1309 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1310 1311 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1312 1313 /* 1314 * Size up per page size free list counters. 1315 */ 1316 ctrs_sz = page_ctrs_sz(); 1317 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1318 1319 if (ctrs_base == NULL) 1320 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1321 1322 /* 1323 * The memseg list is for the chunks of physical memory that 1324 * will be managed by the vm system. The number calculated is 1325 * a guess as boot may fragment it more when memory allocations 1326 * are made before kphysm_init(). Currently, there are two 1327 * allocations before then, so we assume each causes fragmen- 1328 * tation, and add a couple more for good measure. 1329 */ 1330 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1331 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1332 1333 if (memseg_base == NULL) 1334 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1335 1336 1337 if (kpm_enable) { 1338 /* 1339 * kpm page space -- Update kpm_npages and make the 1340 * same assumption about fragmenting as it is done 1341 * for memseg_sz above. 1342 */ 1343 kpm_npages_setup(memblocks + 4); 1344 kpm_pp_sz = (kpm_smallpages == 0) ? 1345 kpm_npages * sizeof (kpm_page_t): 1346 kpm_npages * sizeof (kpm_spage_t); 1347 1348 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1349 ecache_alignsize); 1350 1351 if (kpm_pp_base == NULL) 1352 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1353 kpm_pp_sz; 1354 } 1355 1356 if (alloc_sz > 0) { 1357 uintptr_t bop_base; 1358 1359 /* 1360 * We need extra memory allocated through BOP_ALLOC. 1361 */ 1362 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1363 alloc_alignsize); 1364 1365 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1366 1367 bop_base = (uintptr_t)alloc_base; 1368 1369 alloc_base += alloc_sz; 1370 1371 if (page_hash == NULL) { 1372 page_hash = (struct page **)bop_base; 1373 bop_base = roundup(bop_base + pagehash_sz, 1374 ecache_alignsize); 1375 } 1376 1377 if (ctrs_base == NULL) { 1378 ctrs_base = (caddr_t)bop_base; 1379 bop_base = roundup(bop_base + ctrs_sz, 1380 ecache_alignsize); 1381 } 1382 1383 if (memseg_base == NULL) { 1384 memseg_base = (struct memseg *)bop_base; 1385 bop_base = roundup(bop_base + memseg_sz, 1386 ecache_alignsize); 1387 } 1388 1389 if (kpm_enable && kpm_pp_base == NULL) { 1390 kpm_pp_base = (uintptr_t)bop_base; 1391 bop_base = roundup(bop_base + kpm_pp_sz, 1392 ecache_alignsize); 1393 } 1394 1395 ASSERT(bop_base <= (uintptr_t)alloc_base); 1396 } 1397 1398 PRM_DEBUG(page_hash); 1399 PRM_DEBUG(memseg_base); 1400 PRM_DEBUG(kpm_pp_base); 1401 PRM_DEBUG(kpm_pp_sz); 1402 PRM_DEBUG(pp_base); 1403 PRM_DEBUG(pp_sz); 1404 PRM_DEBUG(alloc_base); 1405 1406 #ifdef TRAPTRACE 1407 alloc_base = trap_trace_alloc(alloc_base); 1408 PRM_DEBUG(alloc_base); 1409 #endif /* TRAPTRACE */ 1410 1411 /* 1412 * In theory it's possible that kmem64 chunk is 0 sized 1413 * (on very small machines). Check for that. 1414 */ 1415 if (alloc_base == kmem64_base) { 1416 kmem64_base = NULL; 1417 kmem64_end = NULL; 1418 kmem64_aligned_end = NULL; 1419 goto kmem64_alloced; 1420 } 1421 1422 /* 1423 * Allocate kmem64 memory. 1424 * Round up to end of large page and overmap. 1425 * kmem64_end..kmem64_aligned_end is added to memory list for reuse 1426 */ 1427 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1428 MMU_PAGESIZE); 1429 1430 /* 1431 * Make one large memory alloc after figuring out the 64-bit size. This 1432 * will enable use of the largest page size appropriate for the system 1433 * architecture. 1434 */ 1435 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 1436 ASSERT(IS_P2ALIGNED(kmem64_base, TTEBYTES(max_bootlp_tteszc))); 1437 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 1438 size_t asize; 1439 #if !defined(C_OBP) 1440 unsigned long long pa; 1441 #endif /* !C_OBP */ 1442 1443 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 1444 continue; 1445 kmem64_alignsize = TTEBYTES(i); 1446 kmem64_szc = i; 1447 1448 /* limit page size for small memory */ 1449 if (mmu_btop(kmem64_alignsize) > (npages >> 2)) 1450 continue; 1451 1452 kmem64_aligned_end = (caddr_t)roundup((uintptr_t)kmem64_end, 1453 kmem64_alignsize); 1454 asize = kmem64_aligned_end - kmem64_base; 1455 #if !defined(C_OBP) 1456 if (prom_allocate_phys(asize, kmem64_alignsize, &pa) == 0) { 1457 if (prom_claim_virt(asize, kmem64_base) != 1458 (caddr_t)-1) { 1459 kmem64_pabase = pa; 1460 install_kmem64_tte(); 1461 break; 1462 } else { 1463 prom_free_phys(asize, pa); 1464 } 1465 } 1466 #else /* !C_OBP */ 1467 if ((caddr_t)BOP_ALLOC(bootops, kmem64_base, asize, 1468 kmem64_alignsize) == kmem64_base) { 1469 kmem64_pabase = va_to_pa(kmem64_base); 1470 break; 1471 } 1472 #endif /* !C_OBP */ 1473 if (i == TTE8K) { 1474 prom_panic("kmem64 allocation failure"); 1475 } 1476 } 1477 1478 PRM_DEBUG(kmem64_base); 1479 PRM_DEBUG(kmem64_end); 1480 PRM_DEBUG(kmem64_aligned_end); 1481 PRM_DEBUG(kmem64_alignsize); 1482 1483 /* 1484 * Now set pa using saved va from above. 1485 */ 1486 if (&ecache_init_scrub_flush_area) { 1487 (void) ecache_init_scrub_flush_area(NULL); 1488 } 1489 1490 kmem64_alloced: 1491 1492 /* 1493 * Initialize per page size free list counters. 1494 */ 1495 ctrs_end = page_ctrs_alloc(ctrs_base); 1496 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1497 1498 /* 1499 * Allocate space for the interrupt vector table and also for the 1500 * reserved interrupt vector data structures. 1501 */ 1502 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 1503 IVSIZE, MMU_PAGESIZE); 1504 if (memspace != (caddr_t)intr_vec_table) 1505 prom_panic("interrupt vector table allocation failure"); 1506 1507 /* 1508 * The memory lists from boot are allocated from the heap arena 1509 * so that later they can be freed and/or reallocated. 1510 */ 1511 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1512 prom_panic("could not retrieve property \"extent\""); 1513 1514 /* 1515 * Between now and when we finish copying in the memory lists, 1516 * allocations happen so the space gets fragmented and the 1517 * lists longer. Leave enough space for lists twice as long 1518 * as what boot says it has now; roundup to a pagesize. 1519 * Also add space for the final phys-avail copy in the fixup 1520 * routine. 1521 */ 1522 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1523 roundup(IVSIZE, MMU_PAGESIZE)); 1524 memlist_sz *= 4; 1525 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1526 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1527 if (memspace == NULL) 1528 halt("Boot allocation failed."); 1529 1530 memlist = (struct memlist *)memspace; 1531 memlist_end = (char *)memspace + memlist_sz; 1532 1533 PRM_DEBUG(memlist); 1534 PRM_DEBUG(memlist_end); 1535 PRM_DEBUG(sysbase); 1536 PRM_DEBUG(syslimit); 1537 1538 kernelheap_init((void *)sysbase, (void *)syslimit, 1539 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1540 1541 /* 1542 * Take the most current snapshot we can by calling mem-update. 1543 */ 1544 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1545 &boot_physavail, &boot_physavail_len, 1546 &boot_virtavail, &boot_virtavail_len); 1547 1548 /* 1549 * Remove the space used by BOP_ALLOC from the kernel heap 1550 * plus the area actually used by the OBP (if any) 1551 * ignoring virtual addresses in virt_avail, above syslimit. 1552 */ 1553 virt_avail = memlist; 1554 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1555 1556 for (cur = virt_avail; cur->next; cur = cur->next) { 1557 uint64_t range_base, range_size; 1558 1559 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1560 continue; 1561 if (range_base >= (uint64_t)syslimit) 1562 break; 1563 /* 1564 * Limit the range to end at syslimit. 1565 */ 1566 range_size = MIN(cur->next->address, 1567 (uint64_t)syslimit) - range_base; 1568 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1569 0, 0, (void *)range_base, (void *)(range_base + range_size), 1570 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1571 } 1572 1573 phys_avail = memlist; 1574 (void) copy_physavail(boot_physavail, boot_physavail_len, 1575 &memlist, 0, 0); 1576 1577 /* 1578 * Add any unused kmem64 memory from overmapped page 1579 * (Note: va_to_pa does not work for kmem64_end) 1580 */ 1581 if (kmem64_end < kmem64_aligned_end) { 1582 uint64_t overlap_size = kmem64_aligned_end - kmem64_end; 1583 uint64_t overlap_pa = kmem64_pabase + 1584 (kmem64_end - kmem64_base); 1585 1586 PRM_DEBUG(overlap_pa); 1587 PRM_DEBUG(overlap_size); 1588 memlist_add(overlap_pa, overlap_size, &memlist, &phys_avail); 1589 } 1590 1591 /* 1592 * Add any extra memory after e_text to the phys_avail list, as long 1593 * as there's at least a page to add. 1594 */ 1595 if (extra_etpg) 1596 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1597 &memlist, &phys_avail); 1598 1599 /* 1600 * Add any extra memory after e_data to the phys_avail list as long 1601 * as there's at least a page to add. Usually, there isn't any, 1602 * since extra HME blocks typically get allocated there first before 1603 * using RAM elsewhere. 1604 */ 1605 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1606 nalloc_base = nalloc_end; 1607 ndata_remain_sz = nalloc_end - nalloc_base; 1608 1609 if (ndata_remain_sz >= MMU_PAGESIZE) 1610 memlist_add(va_to_pa(nalloc_base), 1611 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1612 1613 PRM_DEBUG(memlist); 1614 PRM_DEBUG(memlist_sz); 1615 PRM_DEBUG(memspace); 1616 1617 if ((caddr_t)memlist > (memspace + memlist_sz)) 1618 prom_panic("memlist overflow"); 1619 1620 PRM_DEBUG(pp_base); 1621 PRM_DEBUG(memseg_base); 1622 PRM_DEBUG(npages); 1623 1624 /* 1625 * Initialize the page structures from the memory lists. 1626 */ 1627 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1628 1629 availrmem_initial = availrmem = freemem; 1630 PRM_DEBUG(availrmem); 1631 1632 /* 1633 * Some of the locks depend on page_hashsz being set! 1634 * kmem_init() depends on this; so, keep it here. 1635 */ 1636 page_lock_init(); 1637 1638 /* 1639 * Initialize kernel memory allocator. 1640 */ 1641 kmem_init(); 1642 1643 /* 1644 * Factor in colorequiv to check additional 'equivalent' bins 1645 */ 1646 if (&page_set_colorequiv_arr_cpu != NULL) 1647 page_set_colorequiv_arr_cpu(); 1648 else 1649 page_set_colorequiv_arr(); 1650 1651 /* 1652 * Initialize bp_mapin(). 1653 */ 1654 bp_init(shm_alignment, HAT_STRICTORDER); 1655 1656 /* 1657 * Reserve space for panicbuf, intr_vec_table and reserved interrupt 1658 * vector data structures from the 32-bit heap. 1659 */ 1660 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1661 panicbuf, panicbuf + PANICBUFSIZE, 1662 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1663 1664 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1665 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1666 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1667 1668 mem_config_init(); 1669 } 1670 1671 static void 1672 startup_modules(void) 1673 { 1674 int proplen, nhblk1, nhblk8; 1675 size_t nhblksz; 1676 pgcnt_t pages_per_hblk; 1677 size_t hme8blk_sz, hme1blk_sz; 1678 1679 /* 1680 * Log any optional messages from the boot program 1681 */ 1682 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1683 if (proplen > 0) { 1684 char *msg; 1685 size_t len = (size_t)proplen; 1686 1687 msg = kmem_zalloc(len, KM_SLEEP); 1688 (void) BOP_GETPROP(bootops, "boot-message", msg); 1689 cmn_err(CE_CONT, "?%s\n", msg); 1690 kmem_free(msg, len); 1691 } 1692 1693 /* 1694 * Let the platforms have a chance to change default 1695 * values before reading system file. 1696 */ 1697 if (&set_platform_defaults) 1698 set_platform_defaults(); 1699 1700 /* 1701 * Calculate default settings of system parameters based upon 1702 * maxusers, yet allow to be overridden via the /etc/system file. 1703 */ 1704 param_calc(0); 1705 1706 mod_setup(); 1707 1708 /* 1709 * If this is a positron, complain and halt. 1710 */ 1711 if (&iam_positron && iam_positron()) { 1712 cmn_err(CE_WARN, "This hardware platform is not supported" 1713 " by this release of Solaris.\n"); 1714 #ifdef DEBUG 1715 prom_enter_mon(); /* Type 'go' to resume */ 1716 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1717 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1718 1719 #else /* DEBUG */ 1720 halt(0); 1721 #endif /* DEBUG */ 1722 } 1723 1724 /* 1725 * If we are running firmware that isn't 64-bit ready 1726 * then complain and halt. 1727 */ 1728 do_prom_version_check(); 1729 1730 /* 1731 * Initialize system parameters 1732 */ 1733 param_init(); 1734 1735 /* 1736 * maxmem is the amount of physical memory we're playing with. 1737 */ 1738 maxmem = physmem; 1739 1740 /* Set segkp limits. */ 1741 ncbase = kdi_segdebugbase; 1742 ncend = kdi_segdebugbase; 1743 1744 /* 1745 * Initialize the hat layer. 1746 */ 1747 hat_init(); 1748 1749 /* 1750 * Initialize segment management stuff. 1751 */ 1752 seg_init(); 1753 1754 /* 1755 * Create the va>tte handler, so the prom can understand 1756 * kernel translations. The handler is installed later, just 1757 * as we are about to take over the trap table from the prom. 1758 */ 1759 create_va_to_tte(); 1760 1761 /* 1762 * Load the forthdebugger (optional) 1763 */ 1764 forthdebug_init(); 1765 1766 /* 1767 * Create OBP node for console input callbacks 1768 * if it is needed. 1769 */ 1770 startup_create_io_node(); 1771 1772 if (modloadonly("fs", "specfs") == -1) 1773 halt("Can't load specfs"); 1774 1775 if (modloadonly("fs", "devfs") == -1) 1776 halt("Can't load devfs"); 1777 1778 if (modloadonly("misc", "swapgeneric") == -1) 1779 halt("Can't load swapgeneric"); 1780 1781 (void) modloadonly("sys", "lbl_edition"); 1782 1783 dispinit(); 1784 1785 /* 1786 * Infer meanings to the members of the idprom buffer. 1787 */ 1788 parse_idprom(); 1789 1790 /* Read cluster configuration data. */ 1791 clconf_init(); 1792 1793 setup_ddi(); 1794 1795 /* 1796 * Lets take this opportunity to load the root device. 1797 */ 1798 if (loadrootmodules() != 0) 1799 debug_enter("Can't load the root filesystem"); 1800 1801 /* 1802 * Load tod driver module for the tod part found on this system. 1803 * Recompute the cpu frequency/delays based on tod as tod part 1804 * tends to keep time more accurately. 1805 */ 1806 if (&load_tod_module) 1807 load_tod_module(); 1808 1809 /* 1810 * Allow platforms to load modules which might 1811 * be needed after bootops are gone. 1812 */ 1813 if (&load_platform_modules) 1814 load_platform_modules(); 1815 1816 setcpudelay(); 1817 1818 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1819 &boot_physavail, &boot_physavail_len, 1820 &boot_virtavail, &boot_virtavail_len); 1821 1822 /* 1823 * Calculation and allocation of hmeblks needed to remap 1824 * the memory allocated by PROM till now. 1825 * Overestimate the number of hblk1 elements by assuming 1826 * worst case of TTE64K mappings. 1827 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1828 */ 1829 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1830 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1831 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1832 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1833 1834 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1835 1836 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1837 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1838 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1839 1840 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1841 1842 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1843 nhblk8 = 0; 1844 while (bop_alloc_pages > 1) { 1845 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1846 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1847 bop_alloc_pages *= hme8blk_sz; 1848 bop_alloc_pages = btopr(bop_alloc_pages); 1849 } 1850 nhblk8 += 2; 1851 1852 /* 1853 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1854 * boundary, the number of hblk8's needed to map the entries in the 1855 * boot_virtavail list needs to be adjusted to take this into 1856 * consideration. Thus, we need to add additional hblk8's since it 1857 * is possible that an hblk8 will not have all 8 slots used due to 1858 * alignment constraints. Since there were boot_virtavail_len entries 1859 * in that list, we need to add that many hblk8's to the number 1860 * already calculated to make sure we don't underestimate. 1861 */ 1862 nhblk8 += boot_virtavail_len; 1863 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1864 1865 /* Allocate in pagesize chunks */ 1866 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1867 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1868 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1869 } 1870 1871 static void 1872 startup_bop_gone(void) 1873 { 1874 extern int bop_io_quiesced; 1875 1876 /* 1877 * Destroy the MD initialized at startup 1878 * The startup initializes the MD framework 1879 * using prom and BOP alloc free it now. 1880 */ 1881 mach_descrip_startup_fini(); 1882 1883 /* 1884 * Call back into boot and release boots resources. 1885 */ 1886 BOP_QUIESCE_IO(bootops); 1887 bop_io_quiesced = 1; 1888 1889 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1890 &boot_physavail, &boot_physavail_len, 1891 &boot_virtavail, &boot_virtavail_len); 1892 /* 1893 * Copy physinstalled list into kernel space. 1894 */ 1895 phys_install = memlist; 1896 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1897 1898 /* 1899 * setup physically contiguous area twice as large as the ecache. 1900 * this is used while doing displacement flush of ecaches 1901 */ 1902 if (&ecache_flush_address) { 1903 ecache_flushaddr = ecache_flush_address(); 1904 if (ecache_flushaddr == (uint64_t)-1) { 1905 cmn_err(CE_PANIC, 1906 "startup: no memory to set ecache_flushaddr"); 1907 } 1908 } 1909 1910 /* 1911 * Virtual available next. 1912 */ 1913 ASSERT(virt_avail != NULL); 1914 memlist_free_list(virt_avail); 1915 virt_avail = memlist; 1916 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1917 1918 /* 1919 * Last chance to ask our booter questions .. 1920 */ 1921 } 1922 1923 1924 /* 1925 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1926 * allocations have been performed. We can't call it in startup_bop_gone 1927 * since later operations can cause obp to allocate more memory. 1928 */ 1929 void 1930 startup_fixup_physavail(void) 1931 { 1932 struct memlist *cur; 1933 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1934 1935 PRM_DEBUG(kmem64_overmap_size); 1936 1937 /* 1938 * take the most current snapshot we can by calling mem-update 1939 */ 1940 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1941 &boot_physavail, &boot_physavail_len, 1942 &boot_virtavail, &boot_virtavail_len); 1943 1944 /* 1945 * Copy phys_avail list, again. 1946 * Both the kernel/boot and the prom have been allocating 1947 * from the original list we copied earlier. 1948 */ 1949 cur = memlist; 1950 (void) copy_physavail(boot_physavail, boot_physavail_len, 1951 &memlist, 0, 0); 1952 1953 /* 1954 * Add any unused kmem64 memory from overmapped page 1955 * (Note: va_to_pa does not work for kmem64_end) 1956 */ 1957 if (kmem64_overmap_size) { 1958 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1959 kmem64_overmap_size, 1960 &memlist, &cur); 1961 } 1962 1963 /* 1964 * Add any extra memory after e_text we added to the phys_avail list 1965 * back to the old list. 1966 */ 1967 if (extra_etpg) 1968 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1969 &memlist, &cur); 1970 if (ndata_remain_sz >= MMU_PAGESIZE) 1971 memlist_add(va_to_pa(nalloc_base), 1972 (uint64_t)ndata_remain_sz, &memlist, &cur); 1973 1974 /* 1975 * There isn't any bounds checking on the memlist area 1976 * so ensure it hasn't overgrown. 1977 */ 1978 if ((caddr_t)memlist > (caddr_t)memlist_end) 1979 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1980 1981 /* 1982 * The kernel removes the pages that were allocated for it from 1983 * the freelist, but we now have to find any -extra- pages that 1984 * the prom has allocated for it's own book-keeping, and remove 1985 * them from the freelist too. sigh. 1986 */ 1987 fix_prom_pages(phys_avail, cur); 1988 1989 ASSERT(phys_avail != NULL); 1990 memlist_free_list(phys_avail); 1991 phys_avail = cur; 1992 1993 /* 1994 * We're done with boot. Just after this point in time, boot 1995 * gets unmapped, so we can no longer rely on its services. 1996 * Zero the bootops to indicate this fact. 1997 */ 1998 bootops = (struct bootops *)NULL; 1999 BOOTOPS_GONE(); 2000 } 2001 2002 static void 2003 startup_vm(void) 2004 { 2005 size_t i; 2006 struct segmap_crargs a; 2007 struct segkpm_crargs b; 2008 2009 uint64_t avmem; 2010 caddr_t va; 2011 pgcnt_t max_phys_segkp; 2012 int mnode; 2013 2014 extern int use_brk_lpg, use_stk_lpg; 2015 2016 /* 2017 * get prom's mappings, create hments for them and switch 2018 * to the kernel context. 2019 */ 2020 hat_kern_setup(); 2021 2022 /* 2023 * Take over trap table 2024 */ 2025 setup_trap_table(); 2026 2027 /* 2028 * Install the va>tte handler, so that the prom can handle 2029 * misses and understand the kernel table layout in case 2030 * we need call into the prom. 2031 */ 2032 install_va_to_tte(); 2033 2034 /* 2035 * Set a flag to indicate that the tba has been taken over. 2036 */ 2037 tba_taken_over = 1; 2038 2039 /* initialize MMU primary context register */ 2040 mmu_init_kcontext(); 2041 2042 /* 2043 * The boot cpu can now take interrupts, x-calls, x-traps 2044 */ 2045 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 2046 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 2047 2048 /* 2049 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 2050 */ 2051 tbr_wr_addr_inited = 1; 2052 2053 /* 2054 * Initialize VM system, and map kernel address space. 2055 */ 2056 kvm_init(); 2057 2058 /* 2059 * XXX4U: previously, we initialized and turned on 2060 * the caches at this point. But of course we have 2061 * nothing to do, as the prom has already done this 2062 * for us -- main memory must be E$able at all times. 2063 */ 2064 2065 /* 2066 * If the following is true, someone has patched 2067 * phsymem to be less than the number of pages that 2068 * the system actually has. Remove pages until system 2069 * memory is limited to the requested amount. Since we 2070 * have allocated page structures for all pages, we 2071 * correct the amount of memory we want to remove 2072 * by the size of the memory used to hold page structures 2073 * for the non-used pages. 2074 */ 2075 if (physmem < npages) { 2076 pgcnt_t diff, off; 2077 struct page *pp; 2078 struct seg kseg; 2079 2080 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 2081 2082 off = 0; 2083 diff = npages - physmem; 2084 diff -= mmu_btopr(diff * sizeof (struct page)); 2085 kseg.s_as = &kas; 2086 while (diff--) { 2087 pp = page_create_va(&unused_pages_vp, (offset_t)off, 2088 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 2089 &kseg, (caddr_t)off); 2090 if (pp == NULL) 2091 cmn_err(CE_PANIC, "limited physmem too much!"); 2092 page_io_unlock(pp); 2093 page_downgrade(pp); 2094 availrmem--; 2095 off += MMU_PAGESIZE; 2096 } 2097 } 2098 2099 /* 2100 * When printing memory, show the total as physmem less 2101 * that stolen by a debugger. 2102 */ 2103 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 2104 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 2105 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 2106 2107 avmem = (uint64_t)freemem << PAGESHIFT; 2108 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 2109 2110 /* 2111 * For small memory systems disable automatic large pages. 2112 */ 2113 if (physmem < privm_lpg_min_physmem) { 2114 use_brk_lpg = 0; 2115 use_stk_lpg = 0; 2116 } 2117 2118 /* 2119 * Perform platform specific freelist processing 2120 */ 2121 if (&plat_freelist_process) { 2122 for (mnode = 0; mnode < max_mem_nodes; mnode++) 2123 if (mem_node_config[mnode].exists) 2124 plat_freelist_process(mnode); 2125 } 2126 2127 /* 2128 * Initialize the segkp segment type. We position it 2129 * after the configured tables and buffers (whose end 2130 * is given by econtig) and before V_WKBASE_ADDR. 2131 * Also in this area is segkmap (size SEGMAPSIZE). 2132 */ 2133 2134 /* XXX - cache alignment? */ 2135 va = (caddr_t)SEGKPBASE; 2136 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2137 2138 max_phys_segkp = (physmem * 2); 2139 2140 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2141 segkpsize = btop(SEGKPDEFSIZE); 2142 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2143 "segkpsize has been reset to %ld pages", segkpsize); 2144 } 2145 2146 i = ptob(MIN(segkpsize, max_phys_segkp)); 2147 2148 rw_enter(&kas.a_lock, RW_WRITER); 2149 if (seg_attach(&kas, va, i, segkp) < 0) 2150 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2151 if (segkp_create(segkp) != 0) 2152 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2153 rw_exit(&kas.a_lock); 2154 2155 /* 2156 * kpm segment 2157 */ 2158 segmap_kpm = kpm_enable && 2159 segmap_kpm && PAGESIZE == MAXBSIZE; 2160 2161 if (kpm_enable) { 2162 rw_enter(&kas.a_lock, RW_WRITER); 2163 2164 /* 2165 * The segkpm virtual range range is larger than the 2166 * actual physical memory size and also covers gaps in 2167 * the physical address range for the following reasons: 2168 * . keep conversion between segkpm and physical addresses 2169 * simple, cheap and unambiguous. 2170 * . avoid extension/shrink of the the segkpm in case of DR. 2171 * . avoid complexity for handling of virtual addressed 2172 * caches, segkpm and the regular mapping scheme must be 2173 * kept in sync wrt. the virtual color of mapped pages. 2174 * Any accesses to virtual segkpm ranges not backed by 2175 * physical memory will fall through the memseg pfn hash 2176 * and will be handled in segkpm_fault. 2177 * Additional kpm_size spaces needed for vac alias prevention. 2178 */ 2179 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2180 segkpm) < 0) 2181 cmn_err(CE_PANIC, "cannot attach segkpm"); 2182 2183 b.prot = PROT_READ | PROT_WRITE; 2184 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2185 2186 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2187 panic("segkpm_create segkpm"); 2188 2189 rw_exit(&kas.a_lock); 2190 2191 mach_kpm_init(); 2192 } 2193 2194 if (!segzio_fromheap) { 2195 size_t size; 2196 size_t physmem_b = mmu_ptob(physmem); 2197 2198 /* size is in bytes, segziosize is in pages */ 2199 if (segziosize == 0) { 2200 size = physmem_b; 2201 } else { 2202 size = mmu_ptob(segziosize); 2203 } 2204 2205 if (size < SEGZIOMINSIZE) { 2206 size = SEGZIOMINSIZE; 2207 } else if (size > SEGZIOMAXSIZE) { 2208 size = SEGZIOMAXSIZE; 2209 /* 2210 * On 64-bit x86, we only have 2TB of KVA. This exists 2211 * for parity with x86. 2212 * 2213 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2214 * doesn't consume all of KVA. However, if we have a 2215 * system that has more thant 512gb of physical memory, 2216 * we can actually consume about half of the difference 2217 * between 512gb and the rest of the available physical 2218 * memory. 2219 */ 2220 if (physmem_b > SEGZIOMAXSIZE) { 2221 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2222 } 2223 } 2224 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2225 /* put the base of the ZIO segment after the kpm segment */ 2226 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2227 PRM_DEBUG(segziosize); 2228 PRM_DEBUG(segzio_base); 2229 2230 /* 2231 * On some platforms, kvm_init is called after the kpm 2232 * sizes have been determined. On SPARC, kvm_init is called 2233 * before, so we have to attach the kzioseg after kvm is 2234 * initialized, otherwise we'll try to allocate from the boot 2235 * area since the kernel heap hasn't yet been configured. 2236 */ 2237 rw_enter(&kas.a_lock, RW_WRITER); 2238 2239 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2240 &kzioseg); 2241 (void) segkmem_zio_create(&kzioseg); 2242 2243 /* create zio area covering new segment */ 2244 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2245 2246 rw_exit(&kas.a_lock); 2247 } 2248 2249 2250 /* 2251 * Now create generic mapping segment. This mapping 2252 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2253 * virtual address is greater than the amount of free 2254 * memory that is available, then we trim back the 2255 * segment size to that amount 2256 */ 2257 va = (caddr_t)SEGMAPBASE; 2258 2259 /* 2260 * 1201049: segkmap base address must be MAXBSIZE aligned 2261 */ 2262 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2263 2264 /* 2265 * Set size of segmap to percentage of freemem at boot, 2266 * but stay within the allowable range 2267 * Note we take percentage before converting from pages 2268 * to bytes to avoid an overflow on 32-bit kernels. 2269 */ 2270 i = mmu_ptob((freemem * segmap_percent) / 100); 2271 2272 if (i < MINMAPSIZE) 2273 i = MINMAPSIZE; 2274 2275 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2276 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2277 2278 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2279 2280 rw_enter(&kas.a_lock, RW_WRITER); 2281 if (seg_attach(&kas, va, i, segkmap) < 0) 2282 cmn_err(CE_PANIC, "cannot attach segkmap"); 2283 2284 a.prot = PROT_READ | PROT_WRITE; 2285 a.shmsize = shm_alignment; 2286 a.nfreelist = 0; /* use segmap driver defaults */ 2287 2288 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2289 panic("segmap_create segkmap"); 2290 rw_exit(&kas.a_lock); 2291 2292 segdev_init(); 2293 } 2294 2295 static void 2296 startup_end(void) 2297 { 2298 if ((caddr_t)memlist > (caddr_t)memlist_end) 2299 panic("memlist overflow 2"); 2300 memlist_free_block((caddr_t)memlist, 2301 ((caddr_t)memlist_end - (caddr_t)memlist)); 2302 memlist = NULL; 2303 2304 /* enable page_relocation since OBP is now done */ 2305 page_relocate_ready = 1; 2306 2307 /* 2308 * Perform tasks that get done after most of the VM 2309 * initialization has been done but before the clock 2310 * and other devices get started. 2311 */ 2312 kern_setup1(); 2313 2314 /* 2315 * Intialize the VM arenas for allocating physically 2316 * contiguus memory chunk for interrupt queues snd 2317 * allocate/register boot cpu's queues, if any and 2318 * allocate dump buffer for sun4v systems to store 2319 * extra crash information during crash dump 2320 */ 2321 contig_mem_init(); 2322 mach_descrip_init(); 2323 cpu_intrq_setup(CPU); 2324 cpu_intrq_register(CPU); 2325 mach_htraptrace_setup(CPU->cpu_id); 2326 mach_htraptrace_configure(CPU->cpu_id); 2327 mach_dump_buffer_init(); 2328 2329 /* 2330 * Initialize interrupt related stuff 2331 */ 2332 cpu_intr_alloc(CPU, NINTR_THREADS); 2333 2334 (void) splzs(); /* allow hi clock ints but not zs */ 2335 2336 /* 2337 * Initialize errors. 2338 */ 2339 error_init(); 2340 2341 /* 2342 * Note that we may have already used kernel bcopy before this 2343 * point - but if you really care about this, adb the use_hw_* 2344 * variables to 0 before rebooting. 2345 */ 2346 mach_hw_copy_limit(); 2347 2348 /* 2349 * Install the "real" preemption guards before DDI services 2350 * are available. 2351 */ 2352 (void) prom_set_preprom(kern_preprom); 2353 (void) prom_set_postprom(kern_postprom); 2354 CPU->cpu_m.mutex_ready = 1; 2355 2356 /* 2357 * Initialize segnf (kernel support for non-faulting loads). 2358 */ 2359 segnf_init(); 2360 2361 /* 2362 * Configure the root devinfo node. 2363 */ 2364 configure(); /* set up devices */ 2365 mach_cpu_halt_idle(); 2366 } 2367 2368 2369 void 2370 post_startup(void) 2371 { 2372 #ifdef PTL1_PANIC_DEBUG 2373 extern void init_ptl1_thread(void); 2374 #endif /* PTL1_PANIC_DEBUG */ 2375 extern void abort_sequence_init(void); 2376 2377 /* 2378 * Set the system wide, processor-specific flags to be passed 2379 * to userland via the aux vector for performance hints and 2380 * instruction set extensions. 2381 */ 2382 bind_hwcap(); 2383 2384 /* 2385 * Startup memory scrubber (if any) 2386 */ 2387 mach_memscrub(); 2388 2389 /* 2390 * Allocate soft interrupt to handle abort sequence. 2391 */ 2392 abort_sequence_init(); 2393 2394 /* 2395 * Configure the rest of the system. 2396 * Perform forceloading tasks for /etc/system. 2397 */ 2398 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2399 /* 2400 * ON4.0: Force /proc module in until clock interrupt handle fixed 2401 * ON4.0: This must be fixed or restated in /etc/systems. 2402 */ 2403 (void) modload("fs", "procfs"); 2404 2405 /* load machine class specific drivers */ 2406 load_mach_drivers(); 2407 2408 /* load platform specific drivers */ 2409 if (&load_platform_drivers) 2410 load_platform_drivers(); 2411 2412 /* load vis simulation module, if we are running w/fpu off */ 2413 if (!fpu_exists) { 2414 if (modload("misc", "vis") == -1) 2415 halt("Can't load vis"); 2416 } 2417 2418 mach_fpras(); 2419 2420 maxmem = freemem; 2421 2422 #ifdef PTL1_PANIC_DEBUG 2423 init_ptl1_thread(); 2424 #endif /* PTL1_PANIC_DEBUG */ 2425 2426 if (&cif_init) 2427 cif_init(); 2428 } 2429 2430 #ifdef PTL1_PANIC_DEBUG 2431 int ptl1_panic_test = 0; 2432 int ptl1_panic_xc_one_test = 0; 2433 int ptl1_panic_xc_all_test = 0; 2434 int ptl1_panic_xt_one_test = 0; 2435 int ptl1_panic_xt_all_test = 0; 2436 kthread_id_t ptl1_thread_p = NULL; 2437 kcondvar_t ptl1_cv; 2438 kmutex_t ptl1_mutex; 2439 int ptl1_recurse_count_threshold = 0x40; 2440 int ptl1_recurse_trap_threshold = 0x3d; 2441 extern void ptl1_recurse(int, int); 2442 extern void ptl1_panic_xt(int, int); 2443 2444 /* 2445 * Called once per second by timeout() to wake up 2446 * the ptl1_panic thread to see if it should cause 2447 * a trap to the ptl1_panic() code. 2448 */ 2449 /* ARGSUSED */ 2450 static void 2451 ptl1_wakeup(void *arg) 2452 { 2453 mutex_enter(&ptl1_mutex); 2454 cv_signal(&ptl1_cv); 2455 mutex_exit(&ptl1_mutex); 2456 } 2457 2458 /* 2459 * ptl1_panic cross call function: 2460 * Needed because xc_one() and xc_some() can pass 2461 * 64 bit args but ptl1_recurse() expects ints. 2462 */ 2463 static void 2464 ptl1_panic_xc(void) 2465 { 2466 ptl1_recurse(ptl1_recurse_count_threshold, 2467 ptl1_recurse_trap_threshold); 2468 } 2469 2470 /* 2471 * The ptl1 thread waits for a global flag to be set 2472 * and uses the recurse thresholds to set the stack depth 2473 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2474 * or indirectly via the cross call and cross trap functions. 2475 * 2476 * This is useful testing stack overflows and normal 2477 * ptl1_panic() states with a know stack frame. 2478 * 2479 * ptl1_recurse() is an asm function in ptl1_panic.s that 2480 * sets the {In, Local, Out, and Global} registers to a 2481 * know state on the stack and just prior to causing a 2482 * test ptl1_panic trap. 2483 */ 2484 static void 2485 ptl1_thread(void) 2486 { 2487 mutex_enter(&ptl1_mutex); 2488 while (ptl1_thread_p) { 2489 cpuset_t other_cpus; 2490 int cpu_id; 2491 int my_cpu_id; 2492 int target_cpu_id; 2493 int target_found; 2494 2495 if (ptl1_panic_test) { 2496 ptl1_recurse(ptl1_recurse_count_threshold, 2497 ptl1_recurse_trap_threshold); 2498 } 2499 2500 /* 2501 * Find potential targets for x-call and x-trap, 2502 * if any exist while preempt is disabled we 2503 * start a ptl1_panic if requested via a 2504 * globals. 2505 */ 2506 kpreempt_disable(); 2507 my_cpu_id = CPU->cpu_id; 2508 other_cpus = cpu_ready_set; 2509 CPUSET_DEL(other_cpus, CPU->cpu_id); 2510 target_found = 0; 2511 if (!CPUSET_ISNULL(other_cpus)) { 2512 /* 2513 * Pick the first one 2514 */ 2515 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2516 if (cpu_id == my_cpu_id) 2517 continue; 2518 2519 if (CPU_XCALL_READY(cpu_id)) { 2520 target_cpu_id = cpu_id; 2521 target_found = 1; 2522 break; 2523 } 2524 } 2525 ASSERT(target_found); 2526 2527 if (ptl1_panic_xc_one_test) { 2528 xc_one(target_cpu_id, 2529 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2530 } 2531 if (ptl1_panic_xc_all_test) { 2532 xc_some(other_cpus, 2533 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2534 } 2535 if (ptl1_panic_xt_one_test) { 2536 xt_one(target_cpu_id, 2537 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2538 } 2539 if (ptl1_panic_xt_all_test) { 2540 xt_some(other_cpus, 2541 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2542 } 2543 } 2544 kpreempt_enable(); 2545 (void) timeout(ptl1_wakeup, NULL, hz); 2546 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2547 } 2548 mutex_exit(&ptl1_mutex); 2549 } 2550 2551 /* 2552 * Called during early startup to create the ptl1_thread 2553 */ 2554 void 2555 init_ptl1_thread(void) 2556 { 2557 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2558 &p0, TS_RUN, 0); 2559 } 2560 #endif /* PTL1_PANIC_DEBUG */ 2561 2562 2563 /* 2564 * Add to a memory list. 2565 * start = start of new memory segment 2566 * len = length of new memory segment in bytes 2567 * memlistp = pointer to array of available memory segment structures 2568 * curmemlistp = memory list to which to add segment. 2569 */ 2570 static void 2571 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2572 struct memlist **curmemlistp) 2573 { 2574 struct memlist *new; 2575 2576 new = *memlistp; 2577 new->address = start; 2578 new->size = len; 2579 *memlistp = new + 1; 2580 2581 memlist_insert(new, curmemlistp); 2582 } 2583 2584 /* 2585 * In the case of architectures that support dynamic addition of 2586 * memory at run-time there are two cases where memsegs need to 2587 * be initialized and added to the memseg list. 2588 * 1) memsegs that are constructed at startup. 2589 * 2) memsegs that are constructed at run-time on 2590 * hot-plug capable architectures. 2591 * This code was originally part of the function kphysm_init(). 2592 */ 2593 2594 static void 2595 memseg_list_add(struct memseg *memsegp) 2596 { 2597 struct memseg **prev_memsegp; 2598 pgcnt_t num; 2599 2600 /* insert in memseg list, decreasing number of pages order */ 2601 2602 num = MSEG_NPAGES(memsegp); 2603 2604 for (prev_memsegp = &memsegs; *prev_memsegp; 2605 prev_memsegp = &((*prev_memsegp)->next)) { 2606 if (num > MSEG_NPAGES(*prev_memsegp)) 2607 break; 2608 } 2609 2610 memsegp->next = *prev_memsegp; 2611 *prev_memsegp = memsegp; 2612 2613 if (kpm_enable) { 2614 memsegp->nextpa = (memsegp->next) ? 2615 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2616 2617 if (prev_memsegp != &memsegs) { 2618 struct memseg *msp; 2619 msp = (struct memseg *)((caddr_t)prev_memsegp - 2620 offsetof(struct memseg, next)); 2621 msp->nextpa = va_to_pa(memsegp); 2622 } else { 2623 memsegspa = va_to_pa(memsegs); 2624 } 2625 } 2626 } 2627 2628 /* 2629 * PSM add_physmem_cb(). US-II and newer processors have some 2630 * flavor of the prefetch capability implemented. We exploit 2631 * this capability for optimum performance. 2632 */ 2633 #define PREFETCH_BYTES 64 2634 2635 void 2636 add_physmem_cb(page_t *pp, pfn_t pnum) 2637 { 2638 extern void prefetch_page_w(void *); 2639 2640 pp->p_pagenum = pnum; 2641 2642 /* 2643 * Prefetch one more page_t into E$. To prevent future 2644 * mishaps with the sizeof(page_t) changing on us, we 2645 * catch this on debug kernels if we can't bring in the 2646 * entire hpage with 2 PREFETCH_BYTES reads. See 2647 * also, sun4u/cpu/cpu_module.c 2648 */ 2649 /*LINTED*/ 2650 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2651 prefetch_page_w((char *)pp); 2652 } 2653 2654 /* 2655 * kphysm_init() tackles the problem of initializing physical memory. 2656 * The old startup made some assumptions about the kernel living in 2657 * physically contiguous space which is no longer valid. 2658 */ 2659 static void 2660 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2661 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2662 { 2663 struct memlist *pmem; 2664 struct memseg *msp; 2665 pfn_t base; 2666 pgcnt_t num; 2667 pfn_t lastseg_pages_end = 0; 2668 pgcnt_t nelem_used = 0; 2669 2670 ASSERT(page_hash != NULL && page_hashsz != 0); 2671 2672 msp = memsegp; 2673 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2674 2675 /* 2676 * Build the memsegs entry 2677 */ 2678 num = btop(pmem->size); 2679 if (num > npages) 2680 num = npages; 2681 npages -= num; 2682 base = btop(pmem->address); 2683 2684 msp->pages = pp; 2685 msp->epages = pp + num; 2686 msp->pages_base = base; 2687 msp->pages_end = base + num; 2688 2689 if (kpm_enable) { 2690 pfn_t pbase_a; 2691 pfn_t pend_a; 2692 pfn_t prev_pend_a; 2693 pgcnt_t nelem; 2694 2695 msp->pagespa = va_to_pa(pp); 2696 msp->epagespa = va_to_pa(pp + num); 2697 pbase_a = kpmptop(ptokpmp(base)); 2698 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2699 nelem = ptokpmp(pend_a - pbase_a); 2700 msp->kpm_nkpmpgs = nelem; 2701 msp->kpm_pbase = pbase_a; 2702 if (lastseg_pages_end) { 2703 /* 2704 * Assume phys_avail is in ascending order 2705 * of physical addresses. 2706 */ 2707 ASSERT(base + num > lastseg_pages_end); 2708 prev_pend_a = kpmptop( 2709 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2710 2711 if (prev_pend_a > pbase_a) { 2712 /* 2713 * Overlap, more than one memseg may 2714 * point to the same kpm_page range. 2715 */ 2716 if (kpm_smallpages == 0) { 2717 msp->kpm_pages = 2718 (kpm_page_t *)kpm_pp - 1; 2719 kpm_pp = (uintptr_t) 2720 ((kpm_page_t *)kpm_pp 2721 + nelem - 1); 2722 } else { 2723 msp->kpm_spages = 2724 (kpm_spage_t *)kpm_pp - 1; 2725 kpm_pp = (uintptr_t) 2726 ((kpm_spage_t *)kpm_pp 2727 + nelem - 1); 2728 } 2729 nelem_used += nelem - 1; 2730 2731 } else { 2732 if (kpm_smallpages == 0) { 2733 msp->kpm_pages = 2734 (kpm_page_t *)kpm_pp; 2735 kpm_pp = (uintptr_t) 2736 ((kpm_page_t *)kpm_pp 2737 + nelem); 2738 } else { 2739 msp->kpm_spages = 2740 (kpm_spage_t *)kpm_pp; 2741 kpm_pp = (uintptr_t) 2742 ((kpm_spage_t *) 2743 kpm_pp + nelem); 2744 } 2745 nelem_used += nelem; 2746 } 2747 2748 } else { 2749 if (kpm_smallpages == 0) { 2750 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2751 kpm_pp = (uintptr_t) 2752 ((kpm_page_t *)kpm_pp + nelem); 2753 } else { 2754 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2755 kpm_pp = (uintptr_t) 2756 ((kpm_spage_t *)kpm_pp + nelem); 2757 } 2758 nelem_used = nelem; 2759 } 2760 2761 if (nelem_used > kpm_npages) 2762 panic("kphysm_init: kpm_pp overflow\n"); 2763 2764 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2765 lastseg_pages_end = msp->pages_end; 2766 } 2767 2768 memseg_list_add(msp); 2769 2770 /* 2771 * add_physmem() initializes the PSM part of the page 2772 * struct by calling the PSM back with add_physmem_cb(). 2773 * In addition it coalesces pages into larger pages as 2774 * it initializes them. 2775 */ 2776 add_physmem(pp, num, base); 2777 pp += num; 2778 msp++; 2779 } 2780 2781 build_pfn_hash(); 2782 } 2783 2784 /* 2785 * Kernel VM initialization. 2786 * Assumptions about kernel address space ordering: 2787 * (1) gap (user space) 2788 * (2) kernel text 2789 * (3) kernel data/bss 2790 * (4) gap 2791 * (5) kernel data structures 2792 * (6) gap 2793 * (7) debugger (optional) 2794 * (8) monitor 2795 * (9) gap (possibly null) 2796 * (10) dvma 2797 * (11) devices 2798 */ 2799 static void 2800 kvm_init(void) 2801 { 2802 /* 2803 * Put the kernel segments in kernel address space. 2804 */ 2805 rw_enter(&kas.a_lock, RW_WRITER); 2806 as_avlinit(&kas); 2807 2808 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2809 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2810 (void) segkmem_create(&ktextseg); 2811 2812 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2813 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2814 (void) segkmem_create(&ktexthole); 2815 2816 (void) seg_attach(&kas, (caddr_t)valloc_base, 2817 (size_t)(econtig32 - valloc_base), &kvalloc); 2818 (void) segkmem_create(&kvalloc); 2819 2820 if (kmem64_base) { 2821 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2822 (size_t)(kmem64_end - kmem64_base), &kmem64); 2823 (void) segkmem_create(&kmem64); 2824 } 2825 2826 /* 2827 * We're about to map out /boot. This is the beginning of the 2828 * system resource management transition. We can no longer 2829 * call into /boot for I/O or memory allocations. 2830 */ 2831 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2832 (void) segkmem_create(&kvseg); 2833 hblk_alloc_dynamic = 1; 2834 2835 /* 2836 * we need to preallocate pages for DR operations before enabling large 2837 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2838 */ 2839 memseg_remap_init(); 2840 2841 /* at this point we are ready to use large page heap */ 2842 segkmem_heap_lp_init(); 2843 2844 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2845 &kvseg32); 2846 (void) segkmem_create(&kvseg32); 2847 2848 /* 2849 * Create a segment for the debugger. 2850 */ 2851 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2852 (void) segkmem_create(&kdebugseg); 2853 2854 rw_exit(&kas.a_lock); 2855 } 2856 2857 char obp_tte_str[] = 2858 "h# %x constant MMU_PAGESHIFT " 2859 "h# %x constant TTE8K " 2860 "h# %x constant SFHME_SIZE " 2861 "h# %x constant SFHME_TTE " 2862 "h# %x constant HMEBLK_TAG " 2863 "h# %x constant HMEBLK_NEXT " 2864 "h# %x constant HMEBLK_MISC " 2865 "h# %x constant HMEBLK_HME1 " 2866 "h# %x constant NHMENTS " 2867 "h# %x constant HBLK_SZMASK " 2868 "h# %x constant HBLK_RANGE_SHIFT " 2869 "h# %x constant HMEBP_HBLK " 2870 "h# %x constant HMEBUCKET_SIZE " 2871 "h# %x constant HTAG_SFMMUPSZ " 2872 "h# %x constant HTAG_REHASHSZ " 2873 "h# %x constant mmu_hashcnt " 2874 "h# %p constant uhme_hash " 2875 "h# %p constant khme_hash " 2876 "h# %x constant UHMEHASH_SZ " 2877 "h# %x constant KHMEHASH_SZ " 2878 "h# %p constant KCONTEXT " 2879 "h# %p constant KHATID " 2880 "h# %x constant ASI_MEM " 2881 2882 ": PHYS-X@ ( phys -- data ) " 2883 " ASI_MEM spacex@ " 2884 "; " 2885 2886 ": PHYS-W@ ( phys -- data ) " 2887 " ASI_MEM spacew@ " 2888 "; " 2889 2890 ": PHYS-L@ ( phys -- data ) " 2891 " ASI_MEM spaceL@ " 2892 "; " 2893 2894 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2895 " 3 * MMU_PAGESHIFT + " 2896 "; " 2897 2898 ": TTE_IS_VALID ( ttep -- flag ) " 2899 " PHYS-X@ 0< " 2900 "; " 2901 2902 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2903 " dup TTE8K = if " 2904 " drop HBLK_RANGE_SHIFT " 2905 " else " 2906 " TTE_PAGE_SHIFT " 2907 " then " 2908 "; " 2909 2910 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2911 " tuck >> swap MMU_PAGESHIFT - << " 2912 "; " 2913 2914 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2915 " >> over xor swap ( hash sfmmup ) " 2916 " KHATID <> if ( hash ) " 2917 " UHMEHASH_SZ and ( bucket ) " 2918 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2919 " else ( hash ) " 2920 " KHMEHASH_SZ and ( bucket ) " 2921 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2922 " then ( hmebp ) " 2923 "; " 2924 2925 ": HME_HASH_TABLE_SEARCH " 2926 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2927 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2928 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2929 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2930 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2931 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2932 " else " 2933 " hmeblk_next + phys-x@ false " 2934 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2935 " then " 2936 " else " 2937 " hmeblk_next + phys-x@ false " 2938 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2939 " then " 2940 " else " 2941 " true " 2942 " then " 2943 " until r> drop " 2944 "; " 2945 2946 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2947 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2948 " HTAG_REHASHSZ << or nip ( hblktag ) " 2949 "; " 2950 2951 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2952 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2953 " TTE8K = if ( hmeblkp addr ) " 2954 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2955 " else ( hmeblkp addr ) " 2956 " drop 0 ( hmeblkp 0 ) " 2957 " then ( hmeblkp hme-index ) " 2958 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2959 " SFHME_TTE + ( ttep ) " 2960 "; " 2961 2962 ": unix-tte ( addr cnum -- false | tte-data true ) " 2963 " KCONTEXT = if ( addr ) " 2964 " KHATID ( addr khatid ) " 2965 " else ( addr ) " 2966 " drop false exit ( false ) " 2967 " then " 2968 " ( addr khatid ) " 2969 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2970 " 2dup swap i HME_HASH_SHIFT " 2971 "( addr sfmmup sfmmup addr hmeshift ) " 2972 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2973 " over i 4 pick " 2974 "( addr sfmmup hmebp sfmmup rehash addr ) " 2975 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2976 " HME_HASH_TABLE_SEARCH " 2977 "( addr sfmmup { null | hmeblkp } ) " 2978 " ?dup if ( addr sfmmup hmeblkp ) " 2979 " nip swap HBLK_TO_TTEP ( ttep ) " 2980 " dup TTE_IS_VALID if ( valid-ttep ) " 2981 " PHYS-X@ true ( tte-data true ) " 2982 " else ( invalid-tte ) " 2983 " drop false ( false ) " 2984 " then ( false | tte-data true ) " 2985 " unloop exit ( false | tte-data true ) " 2986 " then ( addr sfmmup ) " 2987 " loop ( addr sfmmup ) " 2988 " 2drop false ( false ) " 2989 "; " 2990 ; 2991 2992 void 2993 create_va_to_tte(void) 2994 { 2995 char *bp; 2996 extern int khmehash_num, uhmehash_num; 2997 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2998 2999 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 3000 3001 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 3002 3003 /* 3004 * Teach obp how to parse our sw ttes. 3005 */ 3006 (void) sprintf(bp, obp_tte_str, 3007 MMU_PAGESHIFT, 3008 TTE8K, 3009 sizeof (struct sf_hment), 3010 OFFSET(struct sf_hment, hme_tte), 3011 OFFSET(struct hme_blk, hblk_tag), 3012 OFFSET(struct hme_blk, hblk_nextpa), 3013 OFFSET(struct hme_blk, hblk_misc), 3014 OFFSET(struct hme_blk, hblk_hme), 3015 NHMENTS, 3016 HBLK_SZMASK, 3017 HBLK_RANGE_SHIFT, 3018 OFFSET(struct hmehash_bucket, hmeh_nextpa), 3019 sizeof (struct hmehash_bucket), 3020 HTAG_SFMMUPSZ, 3021 HTAG_REHASHSZ, 3022 mmu_hashcnt, 3023 (caddr_t)va_to_pa((caddr_t)uhme_hash), 3024 (caddr_t)va_to_pa((caddr_t)khme_hash), 3025 UHMEHASH_SZ, 3026 KHMEHASH_SZ, 3027 KCONTEXT, 3028 KHATID, 3029 ASI_MEM); 3030 prom_interpret(bp, 0, 0, 0, 0, 0); 3031 3032 kobj_free(bp, MMU_PAGESIZE); 3033 } 3034 3035 void 3036 install_va_to_tte(void) 3037 { 3038 /* 3039 * advise prom that he can use unix-tte 3040 */ 3041 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3042 } 3043 3044 3045 /* 3046 * Because kmdb links prom_stdout_is_framebuffer into its own 3047 * module, we add "device-type=display" here for /os-io node, so that 3048 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 3049 * is registered into OBP. 3050 */ 3051 static char *create_node = 3052 "\" /\" find-device " 3053 "new-device " 3054 "\" os-io\" device-name " 3055 "\" display\" device-type " 3056 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3057 " 2>r swap 2 2r> ['] $callback catch if " 3058 " 2drop 3drop 0 " 3059 " then " 3060 "; " 3061 ": read ( adr,len -- #read ) " 3062 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3063 " ( retN ... ret1 N ) " 3064 " ?dup if " 3065 " swap >r 1- 0 ?do drop loop r> " 3066 " else " 3067 " -2 " 3068 " then " 3069 "; " 3070 ": write ( adr,len -- #written ) " 3071 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3072 " ( retN ... ret1 N ) " 3073 " ?dup if " 3074 " swap >r 1- 0 ?do drop loop r> " 3075 " else " 3076 " 0 " 3077 " then " 3078 "; " 3079 ": poll-tty ( -- ) ; " 3080 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3081 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3082 ": cb-give/take ( $method -- ) " 3083 " 0 -rot ['] $callback catch ?dup if " 3084 " >r 2drop 2drop r> throw " 3085 " else " 3086 " 0 ?do drop loop " 3087 " then " 3088 "; " 3089 ": give ( -- ) \" exit-input\" cb-give/take ; " 3090 ": take ( -- ) \" enter-input\" cb-give/take ; " 3091 ": open ( -- ok? ) true ; " 3092 ": close ( -- ) ; " 3093 "finish-device " 3094 "device-end "; 3095 3096 /* 3097 * Create the OBP input/output node (FCode serial driver). 3098 * It is needed for both USB console keyboard and for 3099 * the kernel terminal emulator. It is too early to check for a 3100 * kernel console compatible framebuffer now, so we create this 3101 * so that we're ready if we need to enable kernel terminal emulation. 3102 * 3103 * When the USB software takes over the input device at the time 3104 * consconfig runs, OBP's stdin is redirected to this node. 3105 * Whenever the FORTH user interface is used after this switch, 3106 * the node will call back into the kernel for console input. 3107 * If a serial device such as ttya or a UART with a Type 5 keyboard 3108 * attached is used, OBP takes over the serial device when the system 3109 * goes to the debugger after the system is booted. This sharing 3110 * of the relatively simple serial device is difficult but possible. 3111 * Sharing the USB host controller is impossible due its complexity. 3112 * 3113 * Similarly to USB keyboard input redirection, after consconfig_dacf 3114 * configures a kernel console framebuffer as the standard output 3115 * device, OBP's stdout is switched to to vector through the 3116 * /os-io node into the kernel terminal emulator. 3117 */ 3118 static void 3119 startup_create_io_node(void) 3120 { 3121 prom_interpret(create_node, 0, 0, 0, 0, 0); 3122 } 3123 3124 3125 static void 3126 do_prom_version_check(void) 3127 { 3128 int i; 3129 pnode_t node; 3130 char buf[64]; 3131 static char drev[] = "Down-rev firmware detected%s\n" 3132 "\tPlease upgrade to the following minimum version:\n" 3133 "\t\t%s\n"; 3134 3135 i = prom_version_check(buf, sizeof (buf), &node); 3136 3137 if (i == PROM_VER64_OK) 3138 return; 3139 3140 if (i == PROM_VER64_UPGRADE) { 3141 cmn_err(CE_WARN, drev, "", buf); 3142 3143 #ifdef DEBUG 3144 prom_enter_mon(); /* Type 'go' to continue */ 3145 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3146 return; 3147 #else 3148 halt(0); 3149 #endif 3150 } 3151 3152 /* 3153 * The other possibility is that this is a server running 3154 * good firmware, but down-rev firmware was detected on at 3155 * least one other cpu board. We just complain if we see 3156 * that. 3157 */ 3158 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3159 } 3160 3161 static void 3162 kpm_init() 3163 { 3164 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 3165 kpm_pgsz = 1ull << kpm_pgshft; 3166 kpm_pgoff = kpm_pgsz - 1; 3167 kpmp2pshft = kpm_pgshft - PAGESHIFT; 3168 kpmpnpgs = 1 << kpmp2pshft; 3169 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 3170 } 3171 3172 void 3173 kpm_npages_setup(int memblocks) 3174 { 3175 /* 3176 * npages can be scattered in a maximum of 'memblocks' 3177 */ 3178 kpm_npages = ptokpmpr(npages) + memblocks; 3179 } 3180 3181 /* 3182 * Must be defined in platform dependent code. 3183 */ 3184 extern caddr_t modtext; 3185 extern size_t modtext_sz; 3186 extern caddr_t moddata; 3187 3188 #define HEAPTEXT_ARENA(addr) \ 3189 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3190 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3191 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3192 3193 #define HEAPTEXT_OVERSIZED(addr) \ 3194 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3195 3196 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3197 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3198 kmutex_t texthole_lock; 3199 3200 char kern_bootargs[OBP_MAXPATHLEN]; 3201 3202 void 3203 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3204 { 3205 uintptr_t addr, limit; 3206 3207 addr = HEAPTEXT_BASE; 3208 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3209 3210 /* 3211 * Before we initialize the text_arena, we want to punch holes in the 3212 * underlying heaptext_arena. This guarantees that for any text 3213 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3214 */ 3215 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3216 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3217 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3218 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3219 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3220 } 3221 3222 /* 3223 * Allocate one page at the oversize to break up the text region 3224 * from the oversized region. 3225 */ 3226 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3227 (void *)limit, (void *)(limit + PAGESIZE), 3228 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3229 3230 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3231 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3232 heaptext_arena, 0, VM_SLEEP); 3233 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3234 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3235 } 3236 3237 caddr_t 3238 kobj_text_alloc(vmem_t *arena, size_t size) 3239 { 3240 caddr_t rval, better; 3241 3242 /* 3243 * First, try a sleeping allocation. 3244 */ 3245 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3246 3247 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3248 return (rval); 3249 3250 /* 3251 * We didn't get the area that we wanted. We're going to try to do an 3252 * allocation with explicit constraints. 3253 */ 3254 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3255 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3256 VM_NOSLEEP | VM_BESTFIT); 3257 3258 if (better != NULL) { 3259 /* 3260 * That worked. Free our first attempt and return. 3261 */ 3262 vmem_free(arena, rval, size); 3263 return (better); 3264 } 3265 3266 /* 3267 * That didn't work; we'll have to return our first attempt. 3268 */ 3269 return (rval); 3270 } 3271 3272 caddr_t 3273 kobj_texthole_alloc(caddr_t addr, size_t size) 3274 { 3275 int arena = HEAPTEXT_ARENA(addr); 3276 char c[30]; 3277 uintptr_t base; 3278 3279 if (HEAPTEXT_OVERSIZED(addr)) { 3280 /* 3281 * If this is an oversized allocation, there is no text hole 3282 * available for it; return NULL. 3283 */ 3284 return (NULL); 3285 } 3286 3287 mutex_enter(&texthole_lock); 3288 3289 if (texthole_arena[arena] == NULL) { 3290 ASSERT(texthole_source[arena] == NULL); 3291 3292 if (arena == 0) { 3293 texthole_source[0] = vmem_create("module_text_holesrc", 3294 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3295 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3296 0, VM_SLEEP); 3297 } else { 3298 base = HEAPTEXT_BASE + 3299 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3300 3301 (void) snprintf(c, sizeof (c), 3302 "heaptext_holesrc_%d", arena); 3303 3304 texthole_source[arena] = vmem_create(c, (void *)base, 3305 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3306 0, VM_SLEEP); 3307 } 3308 3309 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3310 3311 texthole_arena[arena] = vmem_create(c, NULL, 0, 3312 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3313 texthole_source[arena], 0, VM_SLEEP); 3314 } 3315 3316 mutex_exit(&texthole_lock); 3317 3318 ASSERT(texthole_arena[arena] != NULL); 3319 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3320 return (vmem_alloc(texthole_arena[arena], size, 3321 VM_BESTFIT | VM_NOSLEEP)); 3322 } 3323 3324 void 3325 kobj_texthole_free(caddr_t addr, size_t size) 3326 { 3327 int arena = HEAPTEXT_ARENA(addr); 3328 3329 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3330 ASSERT(texthole_arena[arena] != NULL); 3331 vmem_free(texthole_arena[arena], addr, size); 3332 } 3333