1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/machsystm.h> 28 #include <sys/archsystm.h> 29 #include <sys/vm.h> 30 #include <sys/cpu.h> 31 #include <sys/atomic.h> 32 #include <sys/reboot.h> 33 #include <sys/kdi.h> 34 #include <sys/bootconf.h> 35 #include <sys/memlist_plat.h> 36 #include <sys/memlist_impl.h> 37 #include <sys/prom_plat.h> 38 #include <sys/prom_isa.h> 39 #include <sys/autoconf.h> 40 #include <sys/ivintr.h> 41 #include <sys/fpu/fpusystm.h> 42 #include <sys/iommutsb.h> 43 #include <vm/vm_dep.h> 44 #include <vm/seg_dev.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_kpm.h> 47 #include <vm/seg_map.h> 48 #include <vm/seg_kp.h> 49 #include <sys/sysconf.h> 50 #include <vm/hat_sfmmu.h> 51 #include <sys/kobj.h> 52 #include <sys/sun4asi.h> 53 #include <sys/clconf.h> 54 #include <sys/platform_module.h> 55 #include <sys/panic.h> 56 #include <sys/cpu_sgnblk_defs.h> 57 #include <sys/clock.h> 58 #include <sys/cmn_err.h> 59 #include <sys/promif.h> 60 #include <sys/prom_debug.h> 61 #include <sys/traptrace.h> 62 #include <sys/memnode.h> 63 #include <sys/mem_cage.h> 64 #include <sys/mmu.h> 65 66 extern void setup_trap_table(void); 67 extern int cpu_intrq_setup(struct cpu *); 68 extern void cpu_intrq_register(struct cpu *); 69 extern void contig_mem_init(void); 70 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 71 extern void mach_dump_buffer_init(void); 72 extern void mach_descrip_init(void); 73 extern void mach_descrip_startup_fini(void); 74 extern void mach_memscrub(void); 75 extern void mach_fpras(void); 76 extern void mach_cpu_halt_idle(void); 77 extern void mach_hw_copy_limit(void); 78 extern void load_mach_drivers(void); 79 extern void load_tod_module(void); 80 #pragma weak load_tod_module 81 82 extern int ndata_alloc_mmfsa(struct memlist *ndata); 83 #pragma weak ndata_alloc_mmfsa 84 85 extern void cif_init(void); 86 #pragma weak cif_init 87 88 extern void parse_idprom(void); 89 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 90 extern void mem_config_init(void); 91 extern void memseg_remap_init(void); 92 93 extern void mach_kpm_init(void); 94 extern void pcf_init(); 95 extern int size_pse_array(pgcnt_t, int); 96 extern void pg_init(); 97 98 /* 99 * External Data: 100 */ 101 extern int vac_size; /* cache size in bytes */ 102 extern uint_t vac_mask; /* VAC alignment consistency mask */ 103 extern uint_t vac_colors; 104 105 /* 106 * Global Data Definitions: 107 */ 108 109 /* 110 * XXX - Don't port this to new architectures 111 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 112 * 'romp' has no use with a prom with an IEEE 1275 client interface. 113 * The driver doesn't use the value, but it depends on the symbol. 114 */ 115 void *romp; /* veritas driver won't load without romp 4154976 */ 116 /* 117 * Declare these as initialized data so we can patch them. 118 */ 119 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 120 pgcnt_t segkpsize = 121 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 122 uint_t segmap_percent = 6; /* Size of segmap segment */ 123 124 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 125 int vac_copyback = 1; 126 char *cache_mode = NULL; 127 int use_mix = 1; 128 int prom_debug = 0; 129 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 151 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 152 caddr_t nalloc_base; /* beginning of nucleus allocation */ 153 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 154 caddr_t valloc_base; /* beginning of kvalloc segment */ 155 156 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 157 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 158 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 159 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 160 int kmem64_szc; /* page size code */ 161 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 162 163 uintptr_t shm_alignment; /* VAC address consistency modulus */ 164 struct memlist *phys_install; /* Total installed physical memory */ 165 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 166 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 167 struct memlist *nopp_list; /* pages with no backing page structs */ 168 struct memlist ndata; /* memlist of nucleus allocatable memory */ 169 int memexp_flag; /* memory expansion card flag */ 170 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 171 pgcnt_t obp_pages; /* Physical pages used by OBP */ 172 173 /* 174 * VM data structures 175 */ 176 long page_hashsz; /* Size of page hash table (power of two) */ 177 struct page *pp_base; /* Base of system page struct array */ 178 size_t pp_sz; /* Size in bytes of page struct array */ 179 struct page **page_hash; /* Page hash table */ 180 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 181 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 182 int pse_shift; /* log2(pse_table_size) */ 183 struct seg ktextseg; /* Segment used for kernel executable image */ 184 struct seg kvalloc; /* Segment used for "valloc" mapping */ 185 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 186 struct seg ktexthole; /* Segment used for nucleus text hole */ 187 struct seg kmapseg; /* Segment used for generic kernel mappings */ 188 struct seg kpmseg; /* Segment used for physical mapping */ 189 struct seg kdebugseg; /* Segment used for the kernel debugger */ 190 191 void *kpm_pp_base; /* Base of system kpm_page array */ 192 size_t kpm_pp_sz; /* Size of system kpm_page array */ 193 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 194 195 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 196 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 197 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 198 199 int segzio_fromheap = 0; /* zio allocations occur from heap */ 200 caddr_t segzio_base; /* Base address of segzio */ 201 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 202 203 /* 204 * A static DR page_t VA map is reserved that can map the page structures 205 * for a domain's entire RA space. The pages that backs this space are 206 * dynamically allocated and need not be physically contiguous. The DR 207 * map size is derived from KPM size. 208 */ 209 int ppvm_enable = 0; /* Static virtual map for page structs */ 210 page_t *ppvm_base; /* Base of page struct map */ 211 pgcnt_t ppvm_size = 0; /* Size of page struct map */ 212 213 /* 214 * debugger pages (if allocated) 215 */ 216 struct vnode kdebugvp; 217 218 /* 219 * VA range available to the debugger 220 */ 221 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 222 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 223 224 /* 225 * Segment for relocated kernel structures in 64-bit large RAM kernels 226 */ 227 struct seg kmem64; 228 229 struct memseg *memseg_free; 230 231 struct vnode unused_pages_vp; 232 233 /* 234 * VM data structures allocated early during boot. 235 */ 236 size_t pagehash_sz; 237 uint64_t memlist_sz; 238 239 char tbr_wr_addr_inited = 0; 240 241 caddr_t mpo_heap32_buf = NULL; 242 size_t mpo_heap32_bufsz = 0; 243 244 /* 245 * Static Routines: 246 */ 247 static int ndata_alloc_memseg(struct memlist *, size_t); 248 static void memlist_new(uint64_t, uint64_t, struct memlist **); 249 static void memlist_add(uint64_t, uint64_t, 250 struct memlist **, struct memlist **); 251 static void kphysm_init(void); 252 static void kvm_init(void); 253 static void install_kmem64_tte(void); 254 255 static void startup_init(void); 256 static void startup_memlist(void); 257 static void startup_modules(void); 258 static void startup_bop_gone(void); 259 static void startup_vm(void); 260 static void startup_end(void); 261 static void setup_cage_params(void); 262 static void startup_create_io_node(void); 263 264 static pgcnt_t npages; 265 static struct memlist *memlist; 266 void *memlist_end; 267 268 static pgcnt_t bop_alloc_pages; 269 static caddr_t hblk_base; 270 uint_t hblk_alloc_dynamic = 0; 271 uint_t hblk1_min = H1MIN; 272 273 274 /* 275 * Hooks for unsupported platforms and down-rev firmware 276 */ 277 int iam_positron(void); 278 #pragma weak iam_positron 279 static void do_prom_version_check(void); 280 281 /* 282 * After receiving a thermal interrupt, this is the number of seconds 283 * to delay before shutting off the system, assuming 284 * shutdown fails. Use /etc/system to change the delay if this isn't 285 * large enough. 286 */ 287 int thermal_powerdown_delay = 1200; 288 289 /* 290 * Used to hold off page relocations into the cage until OBP has completed 291 * its boot-time handoff of its resources to the kernel. 292 */ 293 int page_relocate_ready = 0; 294 295 /* 296 * Indicate if kmem64 allocation was done in small chunks 297 */ 298 int kmem64_smchunks = 0; 299 300 /* 301 * Enable some debugging messages concerning memory usage... 302 */ 303 #ifdef DEBUGGING_MEM 304 static int debugging_mem; 305 static void 306 printmemlist(char *title, struct memlist *list) 307 { 308 if (!debugging_mem) 309 return; 310 311 printf("%s\n", title); 312 313 while (list) { 314 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 315 (uint32_t)(list->ml_address >> 32), 316 (uint32_t)list->ml_address, 317 (uint32_t)(list->ml_size >> 32), 318 (uint32_t)(list->ml_size)); 319 list = list->ml_next; 320 } 321 } 322 323 void 324 printmemseg(struct memseg *memseg) 325 { 326 if (!debugging_mem) 327 return; 328 329 printf("memseg\n"); 330 331 while (memseg) { 332 prom_printf("\tpage = 0x%p, epage = 0x%p, " 333 "pfn = 0x%x, epfn = 0x%x\n", 334 memseg->pages, memseg->epages, 335 memseg->pages_base, memseg->pages_end); 336 memseg = memseg->next; 337 } 338 } 339 340 #define debug_pause(str) halt((str)) 341 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 342 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 343 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 344 #define MPRINTF3(str, a, b, c) \ 345 if (debugging_mem) prom_printf((str), (a), (b), (c)) 346 #else /* DEBUGGING_MEM */ 347 #define MPRINTF(str) 348 #define MPRINTF1(str, a) 349 #define MPRINTF2(str, a, b) 350 #define MPRINTF3(str, a, b, c) 351 #endif /* DEBUGGING_MEM */ 352 353 354 /* 355 * 356 * Kernel's Virtual Memory Layout. 357 * /-----------------------\ 358 * 0xFFFFFFFF.FFFFFFFF -| |- 359 * | OBP's virtual page | 360 * | tables | 361 * 0xFFFFFFFC.00000000 -|-----------------------|- 362 * : : 363 * : : 364 * -|-----------------------|- 365 * | segzio | (base and size vary) 366 * 0xFFFFFE00.00000000 -|-----------------------|- 367 * | | Ultrasparc I/II support 368 * | segkpm segment | up to 2TB of physical 369 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 370 * | | 371 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 372 * : : 373 * : : 374 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 375 * | | ^ 376 * | UltraSPARC I/II call | | 377 * | bug requires an extra | | 378 * | 4 GB of space between | | 379 * | hole and used RAM | | 380 * | | | 381 * 0xFFFFF800.00000000 -|-----------------------|- | 382 * | | | 383 * | Virtual Address Hole | UltraSPARC 384 * | on UltraSPARC I/II | I/II * ONLY * 385 * | | | 386 * 0x00000800.00000000 -|-----------------------|- | 387 * | | | 388 * | UltraSPARC I/II call | | 389 * | bug requires an extra | | 390 * | 4 GB of space between | | 391 * | hole and used RAM | | 392 * | | v 393 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 394 * : : ^ 395 * : : | 396 * |-----------------------| | 397 * | | | 398 * | ecache flush area | | 399 * | (twice largest e$) | | 400 * | | | 401 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 402 * | overmapped area | alignend_end | 403 * | (kmem64_alignsize | | 404 * | boundary) | | 405 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 406 * | | | 407 * | 64-bit kernel ONLY | | 408 * | | | 409 * | kmem64 segment | | 410 * | | | 411 * | (Relocated extra HME | Approximately 412 * | block allocations, | 1 TB of virtual 413 * | memnode freelists, | address space 414 * | HME hash buckets, | | 415 * | mml_table, kpmp_table,| | 416 * | page_t array and | | 417 * | hashblock pool to | | 418 * | avoid hard-coded | | 419 * | 32-bit vaddr | | 420 * | limitations) | | 421 * | | v 422 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 423 * | | 424 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 425 * | | 426 * 0x00000300.00000000 -|-----------------------|- SYSBASE 427 * : : 428 * : : 429 * -|-----------------------|- 430 * | | 431 * | segmap segment | SEGMAPSIZE (1/8th physmem, 432 * | | 256G MAX) 433 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 434 * : : 435 * : : 436 * -|-----------------------|- 437 * | | 438 * | segkp | SEGKPSIZE (2GB) 439 * | | 440 * | | 441 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 442 * | | 443 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 444 * | | (SEGKPBASE - 0x400000) 445 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 446 * | | (MEMSCRUBBASE - NCARGS) 447 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 448 * | | (ARGSBASE - PPMAPSIZE) 449 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 450 * | | 451 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 452 * | | 453 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 454 * : : 455 * : : 456 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 457 * | | 458 * | OBP | 459 * | | 460 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 461 * | kmdb | 462 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 463 * : : 464 * : : 465 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 466 * | | 467 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 468 * | | ~64MB) 469 * -|-----------------------| 470 * | IVSIZE | 471 * 0x00000000.70004000 -|-----------------------| 472 * | panicbuf | 473 * 0x00000000.70002000 -|-----------------------| 474 * | PAGESIZE | 475 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 476 * | boot-time | 477 * | temporary space | 478 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 479 * : : 480 * : : 481 * | | 482 * |-----------------------|- econtig32 483 * | vm structures | 484 * 0x00000000.01C00000 |-----------------------|- nalloc_end 485 * | TSBs | 486 * |-----------------------|- end/nalloc_base 487 * | kernel data & bss | 488 * 0x00000000.01800000 -|-----------------------| 489 * : nucleus text hole : 490 * 0x00000000.01400000 -|-----------------------| 491 * : : 492 * |-----------------------| 493 * | module text | 494 * |-----------------------|- e_text/modtext 495 * | kernel text | 496 * |-----------------------| 497 * | trap table (48k) | 498 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 499 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 500 * |-----------------------| 501 * | | 502 * | invalid | 503 * | | 504 * 0x00000000.00000000 _|_______________________| 505 * 506 * 507 * 508 * 32-bit User Virtual Memory Layout. 509 * /-----------------------\ 510 * | | 511 * | invalid | 512 * | | 513 * 0xFFC00000 -|-----------------------|- USERLIMIT 514 * | user stack | 515 * : : 516 * : : 517 * : : 518 * | user data | 519 * -|-----------------------|- 520 * | user text | 521 * 0x00002000 -|-----------------------|- 522 * | invalid | 523 * 0x00000000 _|_______________________| 524 * 525 * 526 * 527 * 64-bit User Virtual Memory Layout. 528 * /-----------------------\ 529 * | | 530 * | invalid | 531 * | | 532 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 533 * | user stack | 534 * : : 535 * : : 536 * : : 537 * | user data | 538 * -|-----------------------|- 539 * | user text | 540 * 0x00000000.01000000 -|-----------------------|- 541 * | invalid | 542 * 0x00000000.00000000 _|_______________________| 543 */ 544 545 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 546 extern uint64_t ecache_flush_address(void); 547 548 #pragma weak load_platform_modules 549 #pragma weak plat_startup_memlist 550 #pragma weak ecache_init_scrub_flush_area 551 #pragma weak ecache_flush_address 552 553 554 /* 555 * By default the DR Cage is enabled for maximum OS 556 * MPSS performance. Users needing to disable the cage mechanism 557 * can set this variable to zero via /etc/system. 558 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 559 * will result in loss of DR functionality. 560 * Platforms wishing to disable kernel Cage by default 561 * should do so in their set_platform_defaults() routine. 562 */ 563 int kernel_cage_enable = 1; 564 565 static void 566 setup_cage_params(void) 567 { 568 void (*func)(void); 569 570 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 571 if (func != NULL) { 572 (*func)(); 573 return; 574 } 575 576 if (kernel_cage_enable == 0) { 577 return; 578 } 579 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 580 581 if (kcage_on) { 582 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 583 } else { 584 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 585 } 586 587 } 588 589 /* 590 * Machine-dependent startup code 591 */ 592 void 593 startup(void) 594 { 595 startup_init(); 596 if (&startup_platform) 597 startup_platform(); 598 startup_memlist(); 599 startup_modules(); 600 setup_cage_params(); 601 startup_bop_gone(); 602 startup_vm(); 603 startup_end(); 604 } 605 606 struct regs sync_reg_buf; 607 uint64_t sync_tt; 608 609 void 610 sync_handler(void) 611 { 612 struct panic_trap_info ti; 613 int i; 614 615 /* 616 * Prevent trying to talk to the other CPUs since they are 617 * sitting in the prom and won't reply. 618 */ 619 for (i = 0; i < NCPU; i++) { 620 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 621 cpu[i]->cpu_flags &= ~CPU_READY; 622 cpu[i]->cpu_flags |= CPU_QUIESCED; 623 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 624 } 625 } 626 627 /* 628 * We've managed to get here without going through the 629 * normal panic code path. Try and save some useful 630 * information. 631 */ 632 if (!panicstr && (curthread->t_panic_trap == NULL)) { 633 ti.trap_type = sync_tt; 634 ti.trap_regs = &sync_reg_buf; 635 ti.trap_addr = NULL; 636 ti.trap_mmu_fsr = 0x0; 637 638 curthread->t_panic_trap = &ti; 639 } 640 641 /* 642 * If we're re-entering the panic path, update the signature 643 * block so that the SC knows we're in the second part of panic. 644 */ 645 if (panicstr) 646 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 647 648 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 649 panic("sync initiated"); 650 } 651 652 653 static void 654 startup_init(void) 655 { 656 /* 657 * We want to save the registers while we're still in OBP 658 * so that we know they haven't been fiddled with since. 659 * (In principle, OBP can't change them just because it 660 * makes a callback, but we'd rather not depend on that 661 * behavior.) 662 */ 663 char sync_str[] = 664 "warning @ warning off : sync " 665 "%%tl-c %%tstate h# %p x! " 666 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 667 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 668 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 669 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 670 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 671 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 672 "%%y h# %p l! %%tl-c %%tt h# %p x! " 673 "sync ; warning !"; 674 675 /* 676 * 20 == num of %p substrings 677 * 16 == max num of chars %p will expand to. 678 */ 679 char bp[sizeof (sync_str) + 16 * 20]; 680 681 /* 682 * Initialize ptl1 stack for the 1st CPU. 683 */ 684 ptl1_init_cpu(&cpu0); 685 686 /* 687 * Initialize the address map for cache consistent mappings 688 * to random pages; must be done after vac_size is set. 689 */ 690 ppmapinit(); 691 692 /* 693 * Initialize the PROM callback handler. 694 */ 695 init_vx_handler(); 696 697 /* 698 * have prom call sync_callback() to handle the sync and 699 * save some useful information which will be stored in the 700 * core file later. 701 */ 702 (void) sprintf((char *)bp, sync_str, 703 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 704 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 705 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 706 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 707 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 708 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 709 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 710 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 711 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 712 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 713 prom_interpret(bp, 0, 0, 0, 0, 0); 714 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 715 } 716 717 718 size_t 719 calc_pp_sz(pgcnt_t npages) 720 { 721 722 return (npages * sizeof (struct page)); 723 } 724 725 size_t 726 calc_kpmpp_sz(pgcnt_t npages) 727 { 728 729 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 730 kpm_pgsz = 1ull << kpm_pgshft; 731 kpm_pgoff = kpm_pgsz - 1; 732 kpmp2pshft = kpm_pgshft - PAGESHIFT; 733 kpmpnpgs = 1 << kpmp2pshft; 734 735 if (kpm_smallpages == 0) { 736 /* 737 * Avoid fragmentation problems in kphysm_init() 738 * by allocating for all of physical memory 739 */ 740 kpm_npages = ptokpmpr(physinstalled); 741 return (kpm_npages * sizeof (kpm_page_t)); 742 } else { 743 kpm_npages = npages; 744 return (kpm_npages * sizeof (kpm_spage_t)); 745 } 746 } 747 748 size_t 749 calc_pagehash_sz(pgcnt_t npages) 750 { 751 752 /* 753 * The page structure hash table size is a power of 2 754 * such that the average hash chain length is PAGE_HASHAVELEN. 755 */ 756 page_hashsz = npages / PAGE_HASHAVELEN; 757 page_hashsz = 1 << highbit(page_hashsz); 758 return (page_hashsz * sizeof (struct page *)); 759 } 760 761 int testkmem64_smchunks = 0; 762 763 int 764 alloc_kmem64(caddr_t base, caddr_t end) 765 { 766 int i; 767 caddr_t aligned_end = NULL; 768 769 if (testkmem64_smchunks) 770 return (1); 771 772 /* 773 * Make one large memory alloc after figuring out the 64-bit size. This 774 * will enable use of the largest page size appropriate for the system 775 * architecture. 776 */ 777 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 778 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 779 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 780 size_t alloc_size, alignsize; 781 #if !defined(C_OBP) 782 unsigned long long pa; 783 #endif /* !C_OBP */ 784 785 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 786 continue; 787 alignsize = TTEBYTES(i); 788 kmem64_szc = i; 789 790 /* limit page size for small memory */ 791 if (mmu_btop(alignsize) > (npages >> 2)) 792 continue; 793 794 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 795 alloc_size = aligned_end - base; 796 #if !defined(C_OBP) 797 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 798 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 799 kmem64_pabase = pa; 800 kmem64_aligned_end = aligned_end; 801 install_kmem64_tte(); 802 break; 803 } else { 804 prom_free_phys(alloc_size, pa); 805 } 806 } 807 #else /* !C_OBP */ 808 if (prom_alloc(base, alloc_size, alignsize) == base) { 809 kmem64_pabase = va_to_pa(kmem64_base); 810 kmem64_aligned_end = aligned_end; 811 break; 812 } 813 #endif /* !C_OBP */ 814 if (i == TTE8K) { 815 #ifdef sun4v 816 /* return failure to try small allocations */ 817 return (1); 818 #else 819 prom_panic("kmem64 allocation failure"); 820 #endif 821 } 822 } 823 ASSERT(aligned_end != NULL); 824 return (0); 825 } 826 827 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 828 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 829 830 #if !defined(C_OBP) 831 /* 832 * Install a temporary tte handler in OBP for kmem64 area. 833 * 834 * We map kmem64 area with large pages before the trap table is taken 835 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 836 * the same area. Duplicate tlb entries with different page sizes 837 * cause unpredicatble behavior. To avoid this, we don't create 838 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 839 * OBP). Instead, we manage translations with a temporary va>tte-data 840 * handler (kmem64-tte). This handler is replaced by unix-tte when 841 * the trap table is taken over. 842 * 843 * The temporary handler knows the physical address of the kmem64 844 * area. It uses the prom's pgmap@ Forth word for other addresses. 845 * 846 * We have to use BOP_ALLOC() method for C-OBP platforms because 847 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 848 * sun4u platforms. On sun4u we flush tlb after trap table is taken 849 * over if we use large pages for kernel heap and kmem64. Since sun4u 850 * prom (unlike sun4v) calls va>tte-data first for client address 851 * translation prom's ttes for kmem64 can't get into TLB even if we 852 * later switch to prom's trap table again. C-OBP uses 4M pages for 853 * client mappings when possible so on all platforms we get the 854 * benefit from large mappings for kmem64 area immediately during 855 * boot. 856 * 857 * pseudo code: 858 * if (context != 0) { 859 * return false 860 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 861 * tte = tte_template + 862 * (((miss_va & pagemask) - kmem64_base)); 863 * return tte, true 864 * } else { 865 * return pgmap@ result 866 * } 867 */ 868 char kmem64_obp_str[] = 869 "h# %lx constant kmem64-base " 870 "h# %lx constant kmem64-end " 871 "h# %lx constant kmem64-pagemask " 872 "h# %lx constant kmem64-template " 873 874 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 875 " if ( addr ) " 876 " drop false exit then ( false ) " 877 " dup kmem64-base kmem64-end within if ( addr ) " 878 " kmem64-pagemask and ( addr' ) " 879 " kmem64-base - ( addr' ) " 880 " kmem64-template + ( tte ) " 881 " true ( tte true ) " 882 " else ( addr ) " 883 " pgmap@ ( tte ) " 884 " dup 0< if true else drop false then ( tte true | false ) " 885 " then ( tte true | false ) " 886 "; " 887 888 "' kmem64-tte is va>tte-data " 889 ; 890 891 static void 892 install_kmem64_tte() 893 { 894 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 895 tte_t tte; 896 897 PRM_DEBUG(kmem64_pabase); 898 PRM_DEBUG(kmem64_szc); 899 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 900 PROC_DATA | HAT_NOSYNC, kmem64_szc); 901 PRM_DEBUG(tte.ll); 902 (void) sprintf(b, kmem64_obp_str, 903 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 904 ASSERT(strlen(b) < sizeof (b)); 905 prom_interpret(b, 0, 0, 0, 0, 0); 906 } 907 #endif /* !C_OBP */ 908 909 /* 910 * As OBP takes up some RAM when the system boots, pages will already be "lost" 911 * to the system and reflected in npages by the time we see it. 912 * 913 * We only want to allocate kernel structures in the 64-bit virtual address 914 * space on systems with enough RAM to make the overhead of keeping track of 915 * an extra kernel memory segment worthwhile. 916 * 917 * Since OBP has already performed its memory allocations by this point, if we 918 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 919 * memory in the 64-bit virtual address space; otherwise keep allocations 920 * contiguous with we've mapped so far in the 32-bit virtual address space. 921 */ 922 #define MINMOVE_RAM_MB ((size_t)1900) 923 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 924 #define BYTES_TO_MB(b) ((b) / 1048576ul) 925 926 pgcnt_t tune_npages = (pgcnt_t) 927 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 928 929 #pragma weak page_set_colorequiv_arr_cpu 930 extern void page_set_colorequiv_arr_cpu(void); 931 extern void page_set_colorequiv_arr(void); 932 933 static pgcnt_t ramdisk_npages; 934 static struct memlist *old_phys_avail; 935 936 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 937 938 static void 939 startup_memlist(void) 940 { 941 size_t hmehash_sz, pagelist_sz, tt_sz; 942 size_t psetable_sz; 943 caddr_t alloc_base; 944 caddr_t memspace; 945 struct memlist *cur; 946 size_t syslimit = (size_t)SYSLIMIT; 947 size_t sysbase = (size_t)SYSBASE; 948 949 /* 950 * Initialize enough of the system to allow kmem_alloc to work by 951 * calling boot to allocate its memory until the time that 952 * kvm_init is completed. The page structs are allocated after 953 * rounding up end to the nearest page boundary; the memsegs are 954 * initialized and the space they use comes from the kernel heap. 955 * With appropriate initialization, they can be reallocated later 956 * to a size appropriate for the machine's configuration. 957 * 958 * At this point, memory is allocated for things that will never 959 * need to be freed, this used to be "valloced". This allows a 960 * savings as the pages don't need page structures to describe 961 * them because them will not be managed by the vm system. 962 */ 963 964 /* 965 * We're loaded by boot with the following configuration (as 966 * specified in the sun4u/conf/Mapfile): 967 * 968 * text: 4 MB chunk aligned on a 4MB boundary 969 * data & bss: 4 MB chunk aligned on a 4MB boundary 970 * 971 * These two chunks will eventually be mapped by 2 locked 4MB 972 * ttes and will represent the nucleus of the kernel. This gives 973 * us some free space that is already allocated, some or all of 974 * which is made available to kernel module text. 975 * 976 * The free space in the data-bss chunk is used for nucleus 977 * allocatable data structures and we reserve it using the 978 * nalloc_base and nalloc_end variables. This space is currently 979 * being used for hat data structures required for tlb miss 980 * handling operations. We align nalloc_base to a l2 cache 981 * linesize because this is the line size the hardware uses to 982 * maintain cache coherency. 983 * 512K is carved out for module data. 984 */ 985 986 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 987 e_moddata = moddata + MODDATA; 988 nalloc_base = e_moddata; 989 990 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 991 valloc_base = nalloc_base; 992 993 /* 994 * Calculate the start of the data segment. 995 */ 996 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 997 prom_panic("nucleus data overflow"); 998 999 PRM_DEBUG(moddata); 1000 PRM_DEBUG(nalloc_base); 1001 PRM_DEBUG(nalloc_end); 1002 1003 /* 1004 * Remember any slop after e_text so we can give it to the modules. 1005 */ 1006 PRM_DEBUG(e_text); 1007 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1008 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1009 prom_panic("nucleus text overflow"); 1010 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1011 modtext; 1012 PRM_DEBUG(modtext); 1013 PRM_DEBUG(modtext_sz); 1014 1015 init_boot_memlists(); 1016 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1017 &boot_physavail, &boot_physavail_len, 1018 &boot_virtavail, &boot_virtavail_len); 1019 1020 /* 1021 * Remember what the physically available highest page is 1022 * so that dumpsys works properly, and find out how much 1023 * memory is installed. 1024 */ 1025 installed_top_size_memlist_array(boot_physinstalled, 1026 boot_physinstalled_len, &physmax, &physinstalled); 1027 PRM_DEBUG(physinstalled); 1028 PRM_DEBUG(physmax); 1029 1030 /* Fill out memory nodes config structure */ 1031 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1032 1033 /* 1034 * npages is the maximum of available physical memory possible. 1035 * (ie. it will never be more than this) 1036 * 1037 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1038 * using phys_avail will underestimate what will end up being freed. 1039 * A better initial guess is just total memory minus the kernel text 1040 */ 1041 npages = physinstalled - btop(MMU_PAGESIZE4M); 1042 1043 /* 1044 * First allocate things that can go in the nucleus data page 1045 * (fault status, TSBs, dmv, CPUs) 1046 */ 1047 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1048 1049 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1050 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1051 1052 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1053 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1054 1055 if (ndata_alloc_dmv(&ndata) != 0) 1056 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1057 1058 if (ndata_alloc_page_mutexs(&ndata) != 0) 1059 cmn_err(CE_PANIC, 1060 "no more nucleus memory after page free lists alloc"); 1061 1062 if (ndata_alloc_hat(&ndata, npages) != 0) 1063 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1064 1065 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1066 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1067 1068 /* 1069 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1070 * 1071 * There are comments all over the SFMMU code warning of dire 1072 * consequences if the TSBs are moved out of 32-bit space. This 1073 * is largely because the asm code uses "sethi %hi(addr)"-type 1074 * instructions which will not provide the expected result if the 1075 * address is a 64-bit one. 1076 * 1077 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1078 */ 1079 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1080 PRM_DEBUG(alloc_base); 1081 1082 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1083 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1084 PRM_DEBUG(alloc_base); 1085 1086 /* 1087 * Allocate IOMMU TSB array. We do this here so that the physical 1088 * memory gets deducted from the PROM's physical memory list. 1089 */ 1090 alloc_base = iommu_tsb_init(alloc_base); 1091 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1092 PRM_DEBUG(alloc_base); 1093 1094 /* 1095 * Allow for an early allocation of physically contiguous memory. 1096 */ 1097 alloc_base = contig_mem_prealloc(alloc_base, npages); 1098 1099 /* 1100 * Platforms like Starcat and OPL need special structures assigned in 1101 * 32-bit virtual address space because their probing routines execute 1102 * FCode, and FCode can't handle 64-bit virtual addresses... 1103 */ 1104 if (&plat_startup_memlist) { 1105 alloc_base = plat_startup_memlist(alloc_base); 1106 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1107 ecache_alignsize); 1108 PRM_DEBUG(alloc_base); 1109 } 1110 1111 /* 1112 * Save off where the contiguous allocations to date have ended 1113 * in econtig32. 1114 */ 1115 econtig32 = alloc_base; 1116 PRM_DEBUG(econtig32); 1117 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1118 cmn_err(CE_PANIC, "econtig32 too big"); 1119 1120 pp_sz = calc_pp_sz(npages); 1121 PRM_DEBUG(pp_sz); 1122 if (kpm_enable) { 1123 kpm_pp_sz = calc_kpmpp_sz(npages); 1124 PRM_DEBUG(kpm_pp_sz); 1125 } 1126 1127 hmehash_sz = calc_hmehash_sz(npages); 1128 PRM_DEBUG(hmehash_sz); 1129 1130 pagehash_sz = calc_pagehash_sz(npages); 1131 PRM_DEBUG(pagehash_sz); 1132 1133 pagelist_sz = calc_free_pagelist_sz(); 1134 PRM_DEBUG(pagelist_sz); 1135 1136 #ifdef TRAPTRACE 1137 tt_sz = calc_traptrace_sz(); 1138 PRM_DEBUG(tt_sz); 1139 #else 1140 tt_sz = 0; 1141 #endif /* TRAPTRACE */ 1142 1143 /* 1144 * Place the array that protects pp->p_selock in the kmem64 wad. 1145 */ 1146 pse_shift = size_pse_array(npages, max_ncpus); 1147 PRM_DEBUG(pse_shift); 1148 pse_table_size = 1 << pse_shift; 1149 PRM_DEBUG(pse_table_size); 1150 psetable_sz = roundup( 1151 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1152 PRM_DEBUG(psetable_sz); 1153 1154 /* 1155 * Now allocate the whole wad 1156 */ 1157 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1158 pagelist_sz + tt_sz + psetable_sz; 1159 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1160 kmem64_base = (caddr_t)syslimit; 1161 kmem64_end = kmem64_base + kmem64_sz; 1162 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1163 /* 1164 * Attempt for kmem64 to allocate one big 1165 * contiguous chunk of memory failed. 1166 * We get here because we are sun4v. 1167 * We will proceed by breaking up 1168 * the allocation into two attempts. 1169 * First, we allocate kpm_pp_sz, hmehash_sz, 1170 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1171 * one contiguous chunk. This is a much smaller 1172 * chunk and we should get it, if not we panic. 1173 * Note that hmehash and tt need to be physically 1174 * (in the real address sense) contiguous. 1175 * Next, we use bop_alloc_chunk() to 1176 * to allocate the page_t structures. 1177 * This will allow the page_t to be allocated 1178 * in multiple smaller chunks. 1179 * In doing so, the assumption that page_t is 1180 * physically contiguous no longer hold, this is ok 1181 * for sun4v but not for sun4u. 1182 */ 1183 size_t tmp_size; 1184 caddr_t tmp_base; 1185 1186 pp_sz = roundup(pp_sz, PAGESIZE); 1187 1188 /* 1189 * Allocate kpm_pp_sz, hmehash_sz, 1190 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1191 */ 1192 tmp_base = kmem64_base + pp_sz; 1193 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1194 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1195 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1196 prom_panic("kmem64 prom_alloc contig failed"); 1197 PRM_DEBUG(tmp_base); 1198 PRM_DEBUG(tmp_size); 1199 1200 /* 1201 * Allocate the page_ts 1202 */ 1203 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1204 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1205 PRM_DEBUG(kmem64_base); 1206 PRM_DEBUG(pp_sz); 1207 1208 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1209 ASSERT(kmem64_aligned_end >= kmem64_end); 1210 1211 kmem64_smchunks = 1; 1212 } else { 1213 1214 /* 1215 * We need to adjust pp_sz for the normal 1216 * case where kmem64 can allocate one large chunk 1217 */ 1218 if (kpm_smallpages == 0) { 1219 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1220 } else { 1221 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1222 sizeof (kpm_spage_t)); 1223 } 1224 pp_sz = npages * sizeof (struct page); 1225 } 1226 1227 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1228 cmn_err(CE_PANIC, "not enough kmem64 space"); 1229 PRM_DEBUG(kmem64_base); 1230 PRM_DEBUG(kmem64_end); 1231 PRM_DEBUG(kmem64_aligned_end); 1232 1233 /* 1234 * ... and divy it up 1235 */ 1236 alloc_base = kmem64_base; 1237 1238 pp_base = (page_t *)alloc_base; 1239 alloc_base += pp_sz; 1240 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1241 PRM_DEBUG(pp_base); 1242 PRM_DEBUG(npages); 1243 1244 if (kpm_enable) { 1245 kpm_pp_base = alloc_base; 1246 if (kpm_smallpages == 0) { 1247 /* kpm_npages based on physinstalled, don't reset */ 1248 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1249 } else { 1250 kpm_npages = ptokpmpr(npages); 1251 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1252 } 1253 alloc_base += kpm_pp_sz; 1254 alloc_base = 1255 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1256 PRM_DEBUG(kpm_pp_base); 1257 } 1258 1259 alloc_base = alloc_hmehash(alloc_base); 1260 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1261 PRM_DEBUG(alloc_base); 1262 1263 page_hash = (page_t **)alloc_base; 1264 alloc_base += pagehash_sz; 1265 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1266 PRM_DEBUG(page_hash); 1267 1268 alloc_base = alloc_page_freelists(alloc_base); 1269 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1270 PRM_DEBUG(alloc_base); 1271 1272 #ifdef TRAPTRACE 1273 ttrace_buf = alloc_base; 1274 alloc_base += tt_sz; 1275 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1276 PRM_DEBUG(alloc_base); 1277 #endif /* TRAPTRACE */ 1278 1279 pse_mutex = (pad_mutex_t *)alloc_base; 1280 alloc_base += psetable_sz; 1281 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1282 PRM_DEBUG(alloc_base); 1283 1284 /* 1285 * Note that if we use small chunk allocations for 1286 * kmem64, we need to ensure kmem64_end is the same as 1287 * kmem64_aligned_end to prevent subsequent logic from 1288 * trying to reuse the overmapping. 1289 * Otherwise we adjust kmem64_end to what we really allocated. 1290 */ 1291 if (kmem64_smchunks) { 1292 kmem64_end = kmem64_aligned_end; 1293 } else { 1294 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1295 } 1296 kmem64_sz = kmem64_end - kmem64_base; 1297 1298 if (&ecache_init_scrub_flush_area) { 1299 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1300 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1301 } 1302 1303 /* 1304 * If physmem is patched to be non-zero, use it instead of 1305 * the monitor value unless physmem is larger than the total 1306 * amount of memory on hand. 1307 */ 1308 if (physmem == 0 || physmem > npages) 1309 physmem = npages; 1310 1311 /* 1312 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1313 * is mounted as root. This memory is held down by OBP and unlike 1314 * the stub boot_archive is never released. 1315 * 1316 * In order to get things sized correctly on lower memory 1317 * machines (where the memory used by the ramdisk represents 1318 * a significant portion of memory), physmem is adjusted. 1319 * 1320 * This is done by subtracting the ramdisk_size which is set 1321 * to the size of the ramdisk (in Kb) in /etc/system at the 1322 * time the miniroot archive is constructed. 1323 */ 1324 if (root_is_ramdisk == B_TRUE) { 1325 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1326 physmem -= ramdisk_npages; 1327 } 1328 1329 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1330 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1331 1332 /* 1333 * Allocate space for the interrupt vector table. 1334 */ 1335 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1336 if (memspace != (caddr_t)intr_vec_table) 1337 prom_panic("interrupt vector table allocation failure"); 1338 1339 /* 1340 * Between now and when we finish copying in the memory lists, 1341 * allocations happen so the space gets fragmented and the 1342 * lists longer. Leave enough space for lists twice as 1343 * long as we have now; then roundup to a pagesize. 1344 */ 1345 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1346 prom_phys_avail_len() + prom_virt_avail_len()); 1347 memlist_sz *= 2; 1348 memlist_sz = roundup(memlist_sz, PAGESIZE); 1349 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1350 if (memspace == NULL) 1351 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1352 1353 memlist = (struct memlist *)memspace; 1354 memlist_end = (char *)memspace + memlist_sz; 1355 PRM_DEBUG(memlist); 1356 PRM_DEBUG(memlist_end); 1357 1358 PRM_DEBUG(sysbase); 1359 PRM_DEBUG(syslimit); 1360 kernelheap_init((void *)sysbase, (void *)syslimit, 1361 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1362 1363 /* 1364 * Take the most current snapshot we can by calling mem-update. 1365 */ 1366 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1367 &boot_physavail, &boot_physavail_len, 1368 &boot_virtavail, &boot_virtavail_len); 1369 1370 /* 1371 * Remove the space used by prom_alloc from the kernel heap 1372 * plus the area actually used by the OBP (if any) 1373 * ignoring virtual addresses in virt_avail, above syslimit. 1374 */ 1375 virt_avail = memlist; 1376 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1377 1378 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) { 1379 uint64_t range_base, range_size; 1380 1381 if ((range_base = cur->ml_address + cur->ml_size) < 1382 (uint64_t)sysbase) 1383 continue; 1384 if (range_base >= (uint64_t)syslimit) 1385 break; 1386 /* 1387 * Limit the range to end at syslimit. 1388 */ 1389 range_size = MIN(cur->ml_next->ml_address, 1390 (uint64_t)syslimit) - range_base; 1391 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1392 0, 0, (void *)range_base, (void *)(range_base + range_size), 1393 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1394 } 1395 1396 phys_avail = memlist; 1397 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1398 1399 /* 1400 * Add any extra memory at the end of the ndata region if there's at 1401 * least a page to add. There might be a few more pages available in 1402 * the middle of the ndata region, but for now they are ignored. 1403 */ 1404 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1405 if (nalloc_base == NULL) 1406 nalloc_base = nalloc_end; 1407 ndata_remain_sz = nalloc_end - nalloc_base; 1408 1409 /* 1410 * Copy physinstalled list into kernel space. 1411 */ 1412 phys_install = memlist; 1413 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1414 1415 /* 1416 * Create list of physical addrs we don't need pp's for: 1417 * kernel text 4M page 1418 * kernel data 4M page - ndata_remain_sz 1419 * kmem64 pages 1420 * 1421 * NB if adding any pages here, make sure no kpm page 1422 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1423 */ 1424 nopp_list = memlist; 1425 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1426 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1427 &memlist, &nopp_list); 1428 1429 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1430 if (!kmem64_smchunks) 1431 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1432 1433 if ((caddr_t)memlist > (memspace + memlist_sz)) 1434 prom_panic("memlist overflow"); 1435 1436 /* 1437 * Size the pcf array based on the number of cpus in the box at 1438 * boot time. 1439 */ 1440 pcf_init(); 1441 1442 /* 1443 * Initialize the page structures from the memory lists. 1444 */ 1445 kphysm_init(); 1446 1447 availrmem_initial = availrmem = freemem; 1448 PRM_DEBUG(availrmem); 1449 1450 /* 1451 * Some of the locks depend on page_hashsz being set! 1452 * kmem_init() depends on this; so, keep it here. 1453 */ 1454 page_lock_init(); 1455 1456 /* 1457 * Initialize kernel memory allocator. 1458 */ 1459 kmem_init(); 1460 1461 /* 1462 * Factor in colorequiv to check additional 'equivalent' bins 1463 */ 1464 if (&page_set_colorequiv_arr_cpu != NULL) 1465 page_set_colorequiv_arr_cpu(); 1466 else 1467 page_set_colorequiv_arr(); 1468 1469 /* 1470 * Initialize bp_mapin(). 1471 */ 1472 bp_init(shm_alignment, HAT_STRICTORDER); 1473 1474 /* 1475 * Reserve space for MPO mblock structs from the 32-bit heap. 1476 */ 1477 1478 if (mpo_heap32_bufsz > (size_t)0) { 1479 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1480 PAGESIZE, 0, 0, mpo_heap32_buf, 1481 mpo_heap32_buf + mpo_heap32_bufsz, 1482 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1483 } 1484 mem_config_init(); 1485 } 1486 1487 static void 1488 startup_modules(void) 1489 { 1490 int nhblk1, nhblk8; 1491 size_t nhblksz; 1492 pgcnt_t pages_per_hblk; 1493 size_t hme8blk_sz, hme1blk_sz; 1494 1495 /* 1496 * Let the platforms have a chance to change default 1497 * values before reading system file. 1498 */ 1499 if (&set_platform_defaults) 1500 set_platform_defaults(); 1501 1502 /* 1503 * Calculate default settings of system parameters based upon 1504 * maxusers, yet allow to be overridden via the /etc/system file. 1505 */ 1506 param_calc(0); 1507 1508 mod_setup(); 1509 1510 /* 1511 * If this is a positron, complain and halt. 1512 */ 1513 if (&iam_positron && iam_positron()) { 1514 cmn_err(CE_WARN, "This hardware platform is not supported" 1515 " by this release of Solaris.\n"); 1516 #ifdef DEBUG 1517 prom_enter_mon(); /* Type 'go' to resume */ 1518 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1519 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1520 1521 #else /* DEBUG */ 1522 halt(0); 1523 #endif /* DEBUG */ 1524 } 1525 1526 /* 1527 * If we are running firmware that isn't 64-bit ready 1528 * then complain and halt. 1529 */ 1530 do_prom_version_check(); 1531 1532 /* 1533 * Initialize system parameters 1534 */ 1535 param_init(); 1536 1537 /* 1538 * maxmem is the amount of physical memory we're playing with. 1539 */ 1540 maxmem = physmem; 1541 1542 /* Set segkp limits. */ 1543 ncbase = kdi_segdebugbase; 1544 ncend = kdi_segdebugbase; 1545 1546 /* 1547 * Initialize the hat layer. 1548 */ 1549 hat_init(); 1550 1551 /* 1552 * Initialize segment management stuff. 1553 */ 1554 seg_init(); 1555 1556 /* 1557 * Create the va>tte handler, so the prom can understand 1558 * kernel translations. The handler is installed later, just 1559 * as we are about to take over the trap table from the prom. 1560 */ 1561 create_va_to_tte(); 1562 1563 /* 1564 * Load the forthdebugger (optional) 1565 */ 1566 forthdebug_init(); 1567 1568 /* 1569 * Create OBP node for console input callbacks 1570 * if it is needed. 1571 */ 1572 startup_create_io_node(); 1573 1574 if (modloadonly("fs", "specfs") == -1) 1575 halt("Can't load specfs"); 1576 1577 if (modloadonly("fs", "devfs") == -1) 1578 halt("Can't load devfs"); 1579 1580 if (modloadonly("fs", "procfs") == -1) 1581 halt("Can't load procfs"); 1582 1583 if (modloadonly("misc", "swapgeneric") == -1) 1584 halt("Can't load swapgeneric"); 1585 1586 (void) modloadonly("sys", "lbl_edition"); 1587 1588 dispinit(); 1589 1590 /* 1591 * Infer meanings to the members of the idprom buffer. 1592 */ 1593 parse_idprom(); 1594 1595 /* Read cluster configuration data. */ 1596 clconf_init(); 1597 1598 setup_ddi(); 1599 1600 /* 1601 * Lets take this opportunity to load the root device. 1602 */ 1603 if (loadrootmodules() != 0) 1604 debug_enter("Can't load the root filesystem"); 1605 1606 /* 1607 * Load tod driver module for the tod part found on this system. 1608 * Recompute the cpu frequency/delays based on tod as tod part 1609 * tends to keep time more accurately. 1610 */ 1611 if (&load_tod_module) 1612 load_tod_module(); 1613 1614 /* 1615 * Allow platforms to load modules which might 1616 * be needed after bootops are gone. 1617 */ 1618 if (&load_platform_modules) 1619 load_platform_modules(); 1620 1621 setcpudelay(); 1622 1623 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1624 &boot_physavail, &boot_physavail_len, 1625 &boot_virtavail, &boot_virtavail_len); 1626 1627 /* 1628 * Calculation and allocation of hmeblks needed to remap 1629 * the memory allocated by PROM till now. 1630 * Overestimate the number of hblk1 elements by assuming 1631 * worst case of TTE64K mappings. 1632 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1633 */ 1634 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1635 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1636 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1637 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1638 1639 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1640 1641 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1642 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1643 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1644 1645 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1646 1647 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1648 nhblk8 = 0; 1649 while (bop_alloc_pages > 1) { 1650 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1651 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1652 bop_alloc_pages *= hme8blk_sz; 1653 bop_alloc_pages = btopr(bop_alloc_pages); 1654 } 1655 nhblk8 += 2; 1656 1657 /* 1658 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1659 * boundary, the number of hblk8's needed to map the entries in the 1660 * boot_virtavail list needs to be adjusted to take this into 1661 * consideration. Thus, we need to add additional hblk8's since it 1662 * is possible that an hblk8 will not have all 8 slots used due to 1663 * alignment constraints. Since there were boot_virtavail_len entries 1664 * in that list, we need to add that many hblk8's to the number 1665 * already calculated to make sure we don't underestimate. 1666 */ 1667 nhblk8 += boot_virtavail_len; 1668 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1669 1670 /* Allocate in pagesize chunks */ 1671 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1672 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1673 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1674 } 1675 1676 static void 1677 startup_bop_gone(void) 1678 { 1679 1680 /* 1681 * Destroy the MD initialized at startup 1682 * The startup initializes the MD framework 1683 * using prom and BOP alloc free it now. 1684 */ 1685 mach_descrip_startup_fini(); 1686 1687 /* 1688 * We're done with prom allocations. 1689 */ 1690 bop_fini(); 1691 1692 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1693 &boot_physavail, &boot_physavail_len, 1694 &boot_virtavail, &boot_virtavail_len); 1695 1696 /* 1697 * setup physically contiguous area twice as large as the ecache. 1698 * this is used while doing displacement flush of ecaches 1699 */ 1700 if (&ecache_flush_address) { 1701 ecache_flushaddr = ecache_flush_address(); 1702 if (ecache_flushaddr == (uint64_t)-1) { 1703 cmn_err(CE_PANIC, 1704 "startup: no memory to set ecache_flushaddr"); 1705 } 1706 } 1707 1708 /* 1709 * Virtual available next. 1710 */ 1711 ASSERT(virt_avail != NULL); 1712 memlist_free_list(virt_avail); 1713 virt_avail = memlist; 1714 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1715 1716 } 1717 1718 1719 /* 1720 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1721 * allocations have been performed. We can't call it in startup_bop_gone 1722 * since later operations can cause obp to allocate more memory. 1723 */ 1724 void 1725 startup_fixup_physavail(void) 1726 { 1727 struct memlist *cur; 1728 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1729 1730 PRM_DEBUG(kmem64_overmap_size); 1731 1732 /* 1733 * take the most current snapshot we can by calling mem-update 1734 */ 1735 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1736 &boot_physavail, &boot_physavail_len, 1737 &boot_virtavail, &boot_virtavail_len); 1738 1739 /* 1740 * Copy phys_avail list, again. 1741 * Both the kernel/boot and the prom have been allocating 1742 * from the original list we copied earlier. 1743 */ 1744 cur = memlist; 1745 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1746 1747 /* 1748 * Add any unused kmem64 memory from overmapped page 1749 * (Note: va_to_pa does not work for kmem64_end) 1750 */ 1751 if (kmem64_overmap_size) { 1752 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1753 kmem64_overmap_size, &memlist, &cur); 1754 } 1755 1756 /* 1757 * Add any extra memory after e_data we added to the phys_avail list 1758 * back to the old list. 1759 */ 1760 if (ndata_remain_sz >= MMU_PAGESIZE) 1761 memlist_add(va_to_pa(nalloc_base), 1762 (uint64_t)ndata_remain_sz, &memlist, &cur); 1763 1764 /* 1765 * There isn't any bounds checking on the memlist area 1766 * so ensure it hasn't overgrown. 1767 */ 1768 if ((caddr_t)memlist > (caddr_t)memlist_end) 1769 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1770 1771 /* 1772 * The kernel removes the pages that were allocated for it from 1773 * the freelist, but we now have to find any -extra- pages that 1774 * the prom has allocated for it's own book-keeping, and remove 1775 * them from the freelist too. sigh. 1776 */ 1777 sync_memlists(phys_avail, cur); 1778 1779 ASSERT(phys_avail != NULL); 1780 1781 old_phys_avail = phys_avail; 1782 phys_avail = cur; 1783 } 1784 1785 void 1786 update_kcage_ranges(uint64_t addr, uint64_t len) 1787 { 1788 pfn_t base = btop(addr); 1789 pgcnt_t num = btop(len); 1790 int rv; 1791 1792 rv = kcage_range_add(base, num, kcage_startup_dir); 1793 1794 if (rv == ENOMEM) { 1795 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1796 (len == 0 ? 0 : BYTES_TO_MB(len))); 1797 } else if (rv != 0) { 1798 /* catch this in debug kernels */ 1799 ASSERT(0); 1800 1801 cmn_err(CE_WARN, "unexpected kcage_range_add" 1802 " return value %d", rv); 1803 } 1804 } 1805 1806 static void 1807 startup_vm(void) 1808 { 1809 size_t i; 1810 struct segmap_crargs a; 1811 struct segkpm_crargs b; 1812 1813 uint64_t avmem; 1814 caddr_t va; 1815 pgcnt_t max_phys_segkp; 1816 int mnode; 1817 1818 extern int use_brk_lpg, use_stk_lpg; 1819 1820 /* 1821 * get prom's mappings, create hments for them and switch 1822 * to the kernel context. 1823 */ 1824 hat_kern_setup(); 1825 1826 /* 1827 * Take over trap table 1828 */ 1829 setup_trap_table(); 1830 1831 /* 1832 * Install the va>tte handler, so that the prom can handle 1833 * misses and understand the kernel table layout in case 1834 * we need call into the prom. 1835 */ 1836 install_va_to_tte(); 1837 1838 /* 1839 * Set a flag to indicate that the tba has been taken over. 1840 */ 1841 tba_taken_over = 1; 1842 1843 /* initialize MMU primary context register */ 1844 mmu_init_kcontext(); 1845 1846 /* 1847 * The boot cpu can now take interrupts, x-calls, x-traps 1848 */ 1849 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1850 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1851 1852 /* 1853 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1854 */ 1855 tbr_wr_addr_inited = 1; 1856 1857 /* 1858 * Initialize VM system, and map kernel address space. 1859 */ 1860 kvm_init(); 1861 1862 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1863 if (kernel_cage_enable) { 1864 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1865 } 1866 memlist_free_list(old_phys_avail); 1867 1868 /* 1869 * If the following is true, someone has patched 1870 * phsymem to be less than the number of pages that 1871 * the system actually has. Remove pages until system 1872 * memory is limited to the requested amount. Since we 1873 * have allocated page structures for all pages, we 1874 * correct the amount of memory we want to remove 1875 * by the size of the memory used to hold page structures 1876 * for the non-used pages. 1877 */ 1878 if (physmem + ramdisk_npages < npages) { 1879 pgcnt_t diff, off; 1880 struct page *pp; 1881 struct seg kseg; 1882 1883 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1884 1885 off = 0; 1886 diff = npages - (physmem + ramdisk_npages); 1887 diff -= mmu_btopr(diff * sizeof (struct page)); 1888 kseg.s_as = &kas; 1889 while (diff--) { 1890 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1891 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1892 &kseg, (caddr_t)off); 1893 if (pp == NULL) 1894 cmn_err(CE_PANIC, "limited physmem too much!"); 1895 page_io_unlock(pp); 1896 page_downgrade(pp); 1897 availrmem--; 1898 off += MMU_PAGESIZE; 1899 } 1900 } 1901 1902 /* 1903 * When printing memory, show the total as physmem less 1904 * that stolen by a debugger. 1905 */ 1906 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1907 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1908 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1909 1910 avmem = (uint64_t)freemem << PAGESHIFT; 1911 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1912 1913 /* 1914 * For small memory systems disable automatic large pages. 1915 */ 1916 if (physmem < privm_lpg_min_physmem) { 1917 use_brk_lpg = 0; 1918 use_stk_lpg = 0; 1919 } 1920 1921 /* 1922 * Perform platform specific freelist processing 1923 */ 1924 if (&plat_freelist_process) { 1925 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1926 if (mem_node_config[mnode].exists) 1927 plat_freelist_process(mnode); 1928 } 1929 1930 /* 1931 * Initialize the segkp segment type. We position it 1932 * after the configured tables and buffers (whose end 1933 * is given by econtig) and before V_WKBASE_ADDR. 1934 * Also in this area is segkmap (size SEGMAPSIZE). 1935 */ 1936 1937 /* XXX - cache alignment? */ 1938 va = (caddr_t)SEGKPBASE; 1939 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1940 1941 max_phys_segkp = (physmem * 2); 1942 1943 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1944 segkpsize = btop(SEGKPDEFSIZE); 1945 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1946 "segkpsize has been reset to %ld pages", segkpsize); 1947 } 1948 1949 i = ptob(MIN(segkpsize, max_phys_segkp)); 1950 1951 rw_enter(&kas.a_lock, RW_WRITER); 1952 if (seg_attach(&kas, va, i, segkp) < 0) 1953 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1954 if (segkp_create(segkp) != 0) 1955 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1956 rw_exit(&kas.a_lock); 1957 1958 /* 1959 * kpm segment 1960 */ 1961 segmap_kpm = kpm_enable && 1962 segmap_kpm && PAGESIZE == MAXBSIZE; 1963 1964 if (kpm_enable) { 1965 rw_enter(&kas.a_lock, RW_WRITER); 1966 1967 /* 1968 * The segkpm virtual range range is larger than the 1969 * actual physical memory size and also covers gaps in 1970 * the physical address range for the following reasons: 1971 * . keep conversion between segkpm and physical addresses 1972 * simple, cheap and unambiguous. 1973 * . avoid extension/shrink of the the segkpm in case of DR. 1974 * . avoid complexity for handling of virtual addressed 1975 * caches, segkpm and the regular mapping scheme must be 1976 * kept in sync wrt. the virtual color of mapped pages. 1977 * Any accesses to virtual segkpm ranges not backed by 1978 * physical memory will fall through the memseg pfn hash 1979 * and will be handled in segkpm_fault. 1980 * Additional kpm_size spaces needed for vac alias prevention. 1981 */ 1982 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1983 segkpm) < 0) 1984 cmn_err(CE_PANIC, "cannot attach segkpm"); 1985 1986 b.prot = PROT_READ | PROT_WRITE; 1987 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1988 1989 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1990 panic("segkpm_create segkpm"); 1991 1992 rw_exit(&kas.a_lock); 1993 1994 mach_kpm_init(); 1995 } 1996 1997 va = kpm_vbase + (kpm_size * vac_colors); 1998 1999 if (!segzio_fromheap) { 2000 size_t size; 2001 size_t physmem_b = mmu_ptob(physmem); 2002 2003 /* size is in bytes, segziosize is in pages */ 2004 if (segziosize == 0) { 2005 size = physmem_b; 2006 } else { 2007 size = mmu_ptob(segziosize); 2008 } 2009 2010 if (size < SEGZIOMINSIZE) { 2011 size = SEGZIOMINSIZE; 2012 } else if (size > SEGZIOMAXSIZE) { 2013 size = SEGZIOMAXSIZE; 2014 /* 2015 * On 64-bit x86, we only have 2TB of KVA. This exists 2016 * for parity with x86. 2017 * 2018 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2019 * doesn't consume all of KVA. However, if we have a 2020 * system that has more thant 512gb of physical memory, 2021 * we can actually consume about half of the difference 2022 * between 512gb and the rest of the available physical 2023 * memory. 2024 */ 2025 if (physmem_b > SEGZIOMAXSIZE) { 2026 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2027 } 2028 } 2029 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2030 /* put the base of the ZIO segment after the kpm segment */ 2031 segzio_base = va; 2032 va += mmu_ptob(segziosize); 2033 PRM_DEBUG(segziosize); 2034 PRM_DEBUG(segzio_base); 2035 2036 /* 2037 * On some platforms, kvm_init is called after the kpm 2038 * sizes have been determined. On SPARC, kvm_init is called 2039 * before, so we have to attach the kzioseg after kvm is 2040 * initialized, otherwise we'll try to allocate from the boot 2041 * area since the kernel heap hasn't yet been configured. 2042 */ 2043 rw_enter(&kas.a_lock, RW_WRITER); 2044 2045 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2046 &kzioseg); 2047 (void) segkmem_zio_create(&kzioseg); 2048 2049 /* create zio area covering new segment */ 2050 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2051 2052 rw_exit(&kas.a_lock); 2053 } 2054 2055 if (ppvm_enable) { 2056 caddr_t ppvm_max; 2057 2058 /* 2059 * ppvm refers to the static VA space used to map 2060 * the page_t's for dynamically added memory. 2061 * 2062 * ppvm_base should not cross a potential VA hole. 2063 * 2064 * ppvm_size should be large enough to map the 2065 * page_t's needed to manage all of KPM range. 2066 */ 2067 ppvm_size = 2068 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2069 MMU_PAGESIZE); 2070 ppvm_max = (caddr_t)(0ull - ppvm_size); 2071 ppvm_base = (page_t *)va; 2072 2073 if ((caddr_t)ppvm_base <= hole_end) { 2074 cmn_err(CE_WARN, 2075 "Memory DR disabled: invalid DR map base: 0x%p\n", 2076 (void *)ppvm_base); 2077 ppvm_enable = 0; 2078 } else if ((caddr_t)ppvm_base > ppvm_max) { 2079 uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2080 2081 cmn_err(CE_WARN, 2082 "Memory DR disabled: insufficient DR map size:" 2083 " 0x%lx (needed 0x%lx)\n", 2084 ppvm_size - diff, ppvm_size); 2085 ppvm_enable = 0; 2086 } 2087 PRM_DEBUG(ppvm_size); 2088 PRM_DEBUG(ppvm_base); 2089 } 2090 2091 /* 2092 * Now create generic mapping segment. This mapping 2093 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2094 * virtual address is greater than the amount of free 2095 * memory that is available, then we trim back the 2096 * segment size to that amount 2097 */ 2098 va = (caddr_t)SEGMAPBASE; 2099 2100 /* 2101 * 1201049: segkmap base address must be MAXBSIZE aligned 2102 */ 2103 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2104 2105 /* 2106 * Set size of segmap to percentage of freemem at boot, 2107 * but stay within the allowable range 2108 * Note we take percentage before converting from pages 2109 * to bytes to avoid an overflow on 32-bit kernels. 2110 */ 2111 i = mmu_ptob((freemem * segmap_percent) / 100); 2112 2113 if (i < MINMAPSIZE) 2114 i = MINMAPSIZE; 2115 2116 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2117 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2118 2119 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2120 2121 rw_enter(&kas.a_lock, RW_WRITER); 2122 if (seg_attach(&kas, va, i, segkmap) < 0) 2123 cmn_err(CE_PANIC, "cannot attach segkmap"); 2124 2125 a.prot = PROT_READ | PROT_WRITE; 2126 a.shmsize = shm_alignment; 2127 a.nfreelist = 0; /* use segmap driver defaults */ 2128 2129 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2130 panic("segmap_create segkmap"); 2131 rw_exit(&kas.a_lock); 2132 2133 segdev_init(); 2134 } 2135 2136 static void 2137 startup_end(void) 2138 { 2139 if ((caddr_t)memlist > (caddr_t)memlist_end) 2140 panic("memlist overflow 2"); 2141 memlist_free_block((caddr_t)memlist, 2142 ((caddr_t)memlist_end - (caddr_t)memlist)); 2143 memlist = NULL; 2144 2145 /* enable page_relocation since OBP is now done */ 2146 page_relocate_ready = 1; 2147 2148 /* 2149 * Perform tasks that get done after most of the VM 2150 * initialization has been done but before the clock 2151 * and other devices get started. 2152 */ 2153 kern_setup1(); 2154 2155 /* 2156 * Perform CPC initialization for this CPU. 2157 */ 2158 kcpc_hw_init(); 2159 2160 /* 2161 * Intialize the VM arenas for allocating physically 2162 * contiguus memory chunk for interrupt queues snd 2163 * allocate/register boot cpu's queues, if any and 2164 * allocate dump buffer for sun4v systems to store 2165 * extra crash information during crash dump 2166 */ 2167 contig_mem_init(); 2168 mach_descrip_init(); 2169 2170 if (cpu_intrq_setup(CPU)) { 2171 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2172 } 2173 cpu_intrq_register(CPU); 2174 mach_htraptrace_setup(CPU->cpu_id); 2175 mach_htraptrace_configure(CPU->cpu_id); 2176 mach_dump_buffer_init(); 2177 2178 /* 2179 * Initialize interrupt related stuff 2180 */ 2181 cpu_intr_alloc(CPU, NINTR_THREADS); 2182 2183 (void) splzs(); /* allow hi clock ints but not zs */ 2184 2185 /* 2186 * Initialize errors. 2187 */ 2188 error_init(); 2189 2190 /* 2191 * Note that we may have already used kernel bcopy before this 2192 * point - but if you really care about this, adb the use_hw_* 2193 * variables to 0 before rebooting. 2194 */ 2195 mach_hw_copy_limit(); 2196 2197 /* 2198 * Install the "real" preemption guards before DDI services 2199 * are available. 2200 */ 2201 (void) prom_set_preprom(kern_preprom); 2202 (void) prom_set_postprom(kern_postprom); 2203 CPU->cpu_m.mutex_ready = 1; 2204 2205 /* 2206 * Initialize segnf (kernel support for non-faulting loads). 2207 */ 2208 segnf_init(); 2209 2210 /* 2211 * Configure the root devinfo node. 2212 */ 2213 configure(); /* set up devices */ 2214 mach_cpu_halt_idle(); 2215 } 2216 2217 2218 void 2219 post_startup(void) 2220 { 2221 #ifdef PTL1_PANIC_DEBUG 2222 extern void init_ptl1_thread(void); 2223 #endif /* PTL1_PANIC_DEBUG */ 2224 extern void abort_sequence_init(void); 2225 2226 /* 2227 * Set the system wide, processor-specific flags to be passed 2228 * to userland via the aux vector for performance hints and 2229 * instruction set extensions. 2230 */ 2231 bind_hwcap(); 2232 2233 /* 2234 * Startup memory scrubber (if any) 2235 */ 2236 mach_memscrub(); 2237 2238 /* 2239 * Allocate soft interrupt to handle abort sequence. 2240 */ 2241 abort_sequence_init(); 2242 2243 /* 2244 * Configure the rest of the system. 2245 * Perform forceloading tasks for /etc/system. 2246 */ 2247 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2248 /* 2249 * ON4.0: Force /proc module in until clock interrupt handle fixed 2250 * ON4.0: This must be fixed or restated in /etc/systems. 2251 */ 2252 (void) modload("fs", "procfs"); 2253 2254 /* load machine class specific drivers */ 2255 load_mach_drivers(); 2256 2257 /* load platform specific drivers */ 2258 if (&load_platform_drivers) 2259 load_platform_drivers(); 2260 2261 /* load vis simulation module, if we are running w/fpu off */ 2262 if (!fpu_exists) { 2263 if (modload("misc", "vis") == -1) 2264 halt("Can't load vis"); 2265 } 2266 2267 mach_fpras(); 2268 2269 maxmem = freemem; 2270 2271 pg_init(); 2272 2273 #ifdef PTL1_PANIC_DEBUG 2274 init_ptl1_thread(); 2275 #endif /* PTL1_PANIC_DEBUG */ 2276 } 2277 2278 #ifdef PTL1_PANIC_DEBUG 2279 int ptl1_panic_test = 0; 2280 int ptl1_panic_xc_one_test = 0; 2281 int ptl1_panic_xc_all_test = 0; 2282 int ptl1_panic_xt_one_test = 0; 2283 int ptl1_panic_xt_all_test = 0; 2284 kthread_id_t ptl1_thread_p = NULL; 2285 kcondvar_t ptl1_cv; 2286 kmutex_t ptl1_mutex; 2287 int ptl1_recurse_count_threshold = 0x40; 2288 int ptl1_recurse_trap_threshold = 0x3d; 2289 extern void ptl1_recurse(int, int); 2290 extern void ptl1_panic_xt(int, int); 2291 2292 /* 2293 * Called once per second by timeout() to wake up 2294 * the ptl1_panic thread to see if it should cause 2295 * a trap to the ptl1_panic() code. 2296 */ 2297 /* ARGSUSED */ 2298 static void 2299 ptl1_wakeup(void *arg) 2300 { 2301 mutex_enter(&ptl1_mutex); 2302 cv_signal(&ptl1_cv); 2303 mutex_exit(&ptl1_mutex); 2304 } 2305 2306 /* 2307 * ptl1_panic cross call function: 2308 * Needed because xc_one() and xc_some() can pass 2309 * 64 bit args but ptl1_recurse() expects ints. 2310 */ 2311 static void 2312 ptl1_panic_xc(void) 2313 { 2314 ptl1_recurse(ptl1_recurse_count_threshold, 2315 ptl1_recurse_trap_threshold); 2316 } 2317 2318 /* 2319 * The ptl1 thread waits for a global flag to be set 2320 * and uses the recurse thresholds to set the stack depth 2321 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2322 * or indirectly via the cross call and cross trap functions. 2323 * 2324 * This is useful testing stack overflows and normal 2325 * ptl1_panic() states with a know stack frame. 2326 * 2327 * ptl1_recurse() is an asm function in ptl1_panic.s that 2328 * sets the {In, Local, Out, and Global} registers to a 2329 * know state on the stack and just prior to causing a 2330 * test ptl1_panic trap. 2331 */ 2332 static void 2333 ptl1_thread(void) 2334 { 2335 mutex_enter(&ptl1_mutex); 2336 while (ptl1_thread_p) { 2337 cpuset_t other_cpus; 2338 int cpu_id; 2339 int my_cpu_id; 2340 int target_cpu_id; 2341 int target_found; 2342 2343 if (ptl1_panic_test) { 2344 ptl1_recurse(ptl1_recurse_count_threshold, 2345 ptl1_recurse_trap_threshold); 2346 } 2347 2348 /* 2349 * Find potential targets for x-call and x-trap, 2350 * if any exist while preempt is disabled we 2351 * start a ptl1_panic if requested via a 2352 * globals. 2353 */ 2354 kpreempt_disable(); 2355 my_cpu_id = CPU->cpu_id; 2356 other_cpus = cpu_ready_set; 2357 CPUSET_DEL(other_cpus, CPU->cpu_id); 2358 target_found = 0; 2359 if (!CPUSET_ISNULL(other_cpus)) { 2360 /* 2361 * Pick the first one 2362 */ 2363 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2364 if (cpu_id == my_cpu_id) 2365 continue; 2366 2367 if (CPU_XCALL_READY(cpu_id)) { 2368 target_cpu_id = cpu_id; 2369 target_found = 1; 2370 break; 2371 } 2372 } 2373 ASSERT(target_found); 2374 2375 if (ptl1_panic_xc_one_test) { 2376 xc_one(target_cpu_id, 2377 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2378 } 2379 if (ptl1_panic_xc_all_test) { 2380 xc_some(other_cpus, 2381 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2382 } 2383 if (ptl1_panic_xt_one_test) { 2384 xt_one(target_cpu_id, 2385 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2386 } 2387 if (ptl1_panic_xt_all_test) { 2388 xt_some(other_cpus, 2389 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2390 } 2391 } 2392 kpreempt_enable(); 2393 (void) timeout(ptl1_wakeup, NULL, hz); 2394 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2395 } 2396 mutex_exit(&ptl1_mutex); 2397 } 2398 2399 /* 2400 * Called during early startup to create the ptl1_thread 2401 */ 2402 void 2403 init_ptl1_thread(void) 2404 { 2405 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2406 &p0, TS_RUN, 0); 2407 } 2408 #endif /* PTL1_PANIC_DEBUG */ 2409 2410 2411 static void 2412 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2413 { 2414 struct memlist *new; 2415 2416 new = *memlistp; 2417 new->ml_address = start; 2418 new->ml_size = len; 2419 *memlistp = new + 1; 2420 } 2421 2422 /* 2423 * Add to a memory list. 2424 * start = start of new memory segment 2425 * len = length of new memory segment in bytes 2426 * memlistp = pointer to array of available memory segment structures 2427 * curmemlistp = memory list to which to add segment. 2428 */ 2429 static void 2430 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2431 struct memlist **curmemlistp) 2432 { 2433 struct memlist *new = *memlistp; 2434 2435 memlist_new(start, len, memlistp); 2436 memlist_insert(new, curmemlistp); 2437 } 2438 2439 static int 2440 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2441 { 2442 int nseg; 2443 size_t memseg_sz; 2444 struct memseg *msp; 2445 2446 /* 2447 * The memseg list is for the chunks of physical memory that 2448 * will be managed by the vm system. The number calculated is 2449 * a guess as boot may fragment it more when memory allocations 2450 * are made before kphysm_init(). 2451 */ 2452 memseg_sz = (avail + 10) * sizeof (struct memseg); 2453 memseg_sz = roundup(memseg_sz, PAGESIZE); 2454 nseg = memseg_sz / sizeof (struct memseg); 2455 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2456 if (msp == NULL) 2457 return (1); 2458 PRM_DEBUG(memseg_free); 2459 2460 while (nseg--) { 2461 msp->next = memseg_free; 2462 memseg_free = msp; 2463 msp++; 2464 } 2465 return (0); 2466 } 2467 2468 /* 2469 * In the case of architectures that support dynamic addition of 2470 * memory at run-time there are two cases where memsegs need to 2471 * be initialized and added to the memseg list. 2472 * 1) memsegs that are constructed at startup. 2473 * 2) memsegs that are constructed at run-time on 2474 * hot-plug capable architectures. 2475 * This code was originally part of the function kphysm_init(). 2476 */ 2477 2478 static void 2479 memseg_list_add(struct memseg *memsegp) 2480 { 2481 struct memseg **prev_memsegp; 2482 pgcnt_t num; 2483 2484 /* insert in memseg list, decreasing number of pages order */ 2485 2486 num = MSEG_NPAGES(memsegp); 2487 2488 for (prev_memsegp = &memsegs; *prev_memsegp; 2489 prev_memsegp = &((*prev_memsegp)->next)) { 2490 if (num > MSEG_NPAGES(*prev_memsegp)) 2491 break; 2492 } 2493 2494 memsegp->next = *prev_memsegp; 2495 *prev_memsegp = memsegp; 2496 2497 if (kpm_enable) { 2498 memsegp->nextpa = (memsegp->next) ? 2499 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2500 2501 if (prev_memsegp != &memsegs) { 2502 struct memseg *msp; 2503 msp = (struct memseg *)((caddr_t)prev_memsegp - 2504 offsetof(struct memseg, next)); 2505 msp->nextpa = va_to_pa(memsegp); 2506 } else { 2507 memsegspa = va_to_pa(memsegs); 2508 } 2509 } 2510 } 2511 2512 /* 2513 * PSM add_physmem_cb(). US-II and newer processors have some 2514 * flavor of the prefetch capability implemented. We exploit 2515 * this capability for optimum performance. 2516 */ 2517 #define PREFETCH_BYTES 64 2518 2519 void 2520 add_physmem_cb(page_t *pp, pfn_t pnum) 2521 { 2522 extern void prefetch_page_w(void *); 2523 2524 pp->p_pagenum = pnum; 2525 2526 /* 2527 * Prefetch one more page_t into E$. To prevent future 2528 * mishaps with the sizeof(page_t) changing on us, we 2529 * catch this on debug kernels if we can't bring in the 2530 * entire hpage with 2 PREFETCH_BYTES reads. See 2531 * also, sun4u/cpu/cpu_module.c 2532 */ 2533 /*LINTED*/ 2534 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2535 prefetch_page_w((char *)pp); 2536 } 2537 2538 /* 2539 * Find memseg with given pfn 2540 */ 2541 static struct memseg * 2542 memseg_find(pfn_t base, pfn_t *next) 2543 { 2544 struct memseg *seg; 2545 2546 if (next != NULL) 2547 *next = LONG_MAX; 2548 for (seg = memsegs; seg != NULL; seg = seg->next) { 2549 if (base >= seg->pages_base && base < seg->pages_end) 2550 return (seg); 2551 if (next != NULL && seg->pages_base > base && 2552 seg->pages_base < *next) 2553 *next = seg->pages_base; 2554 } 2555 return (NULL); 2556 } 2557 2558 /* 2559 * Put page allocated by OBP on prom_ppages 2560 */ 2561 static void 2562 kphysm_erase(uint64_t addr, uint64_t len) 2563 { 2564 struct page *pp; 2565 struct memseg *seg; 2566 pfn_t base = btop(addr), next; 2567 pgcnt_t num = btop(len); 2568 2569 while (num != 0) { 2570 pgcnt_t off, left; 2571 2572 seg = memseg_find(base, &next); 2573 if (seg == NULL) { 2574 if (next == LONG_MAX) 2575 break; 2576 left = MIN(next - base, num); 2577 base += left, num -= left; 2578 continue; 2579 } 2580 off = base - seg->pages_base; 2581 pp = seg->pages + off; 2582 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2583 while (num != left) { 2584 /* 2585 * init it, lock it, and hashin on prom_pages vp. 2586 * 2587 * Mark it as NONRELOC to let DR know the page 2588 * is locked long term, otherwise DR hangs when 2589 * trying to remove those pages. 2590 * 2591 * XXX vnode offsets on the prom_ppages vnode 2592 * are page numbers (gack) for >32 bit 2593 * physical memory machines. 2594 */ 2595 PP_SETNORELOC(pp); 2596 add_physmem_cb(pp, base); 2597 if (page_trylock(pp, SE_EXCL) == 0) 2598 cmn_err(CE_PANIC, "prom page locked"); 2599 (void) page_hashin(pp, &promvp, 2600 (offset_t)base, NULL); 2601 (void) page_pp_lock(pp, 0, 1); 2602 pp++, base++, num--; 2603 } 2604 } 2605 } 2606 2607 static page_t *ppnext; 2608 static pgcnt_t ppleft; 2609 2610 static void *kpm_ppnext; 2611 static pgcnt_t kpm_ppleft; 2612 2613 /* 2614 * Create a memseg 2615 */ 2616 static void 2617 kphysm_memseg(uint64_t addr, uint64_t len) 2618 { 2619 pfn_t base = btop(addr); 2620 pgcnt_t num = btop(len); 2621 struct memseg *seg; 2622 2623 seg = memseg_free; 2624 memseg_free = seg->next; 2625 ASSERT(seg != NULL); 2626 2627 seg->pages = ppnext; 2628 seg->epages = ppnext + num; 2629 seg->pages_base = base; 2630 seg->pages_end = base + num; 2631 ppnext += num; 2632 ppleft -= num; 2633 2634 if (kpm_enable) { 2635 pgcnt_t kpnum = ptokpmpr(num); 2636 2637 if (kpnum > kpm_ppleft) 2638 panic("kphysm_memseg: kpm_pp overflow"); 2639 seg->pagespa = va_to_pa(seg->pages); 2640 seg->epagespa = va_to_pa(seg->epages); 2641 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2642 seg->kpm_nkpmpgs = kpnum; 2643 /* 2644 * In the kpm_smallpage case, the kpm array 2645 * is 1-1 wrt the page array 2646 */ 2647 if (kpm_smallpages) { 2648 kpm_spage_t *kpm_pp = kpm_ppnext; 2649 2650 kpm_ppnext = kpm_pp + kpnum; 2651 seg->kpm_spages = kpm_pp; 2652 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2653 } else { 2654 kpm_page_t *kpm_pp = kpm_ppnext; 2655 2656 kpm_ppnext = kpm_pp + kpnum; 2657 seg->kpm_pages = kpm_pp; 2658 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2659 /* ASSERT no kpm overlaps */ 2660 ASSERT( 2661 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2662 ASSERT(memseg_find( 2663 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2664 } 2665 kpm_ppleft -= kpnum; 2666 } 2667 2668 memseg_list_add(seg); 2669 } 2670 2671 /* 2672 * Add range to free list 2673 */ 2674 void 2675 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2676 { 2677 struct page *pp; 2678 struct memseg *seg; 2679 pfn_t base = btop(addr); 2680 pgcnt_t num = btop(len); 2681 2682 seg = memseg_find(base, NULL); 2683 ASSERT(seg != NULL); 2684 pp = seg->pages + (base - seg->pages_base); 2685 2686 if (reclaim) { 2687 struct page *rpp = pp; 2688 struct page *lpp = pp + num; 2689 2690 /* 2691 * page should be locked on prom_ppages 2692 * unhash and unlock it 2693 */ 2694 while (rpp < lpp) { 2695 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp); 2696 ASSERT(PP_ISNORELOC(rpp)); 2697 PP_CLRNORELOC(rpp); 2698 page_pp_unlock(rpp, 0, 1); 2699 page_hashout(rpp, NULL); 2700 page_unlock(rpp); 2701 rpp++; 2702 } 2703 } 2704 2705 /* 2706 * add_physmem() initializes the PSM part of the page 2707 * struct by calling the PSM back with add_physmem_cb(). 2708 * In addition it coalesces pages into larger pages as 2709 * it initializes them. 2710 */ 2711 add_physmem(pp, num, base); 2712 } 2713 2714 /* 2715 * kphysm_init() tackles the problem of initializing physical memory. 2716 */ 2717 static void 2718 kphysm_init(void) 2719 { 2720 struct memlist *pmem; 2721 2722 ASSERT(page_hash != NULL && page_hashsz != 0); 2723 2724 ppnext = pp_base; 2725 ppleft = npages; 2726 kpm_ppnext = kpm_pp_base; 2727 kpm_ppleft = kpm_npages; 2728 2729 /* 2730 * installed pages not on nopp_memlist go in memseg list 2731 */ 2732 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2733 2734 /* 2735 * Free the avail list 2736 */ 2737 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next) 2738 kphysm_add(pmem->ml_address, pmem->ml_size, 0); 2739 2740 /* 2741 * Erase pages that aren't available 2742 */ 2743 diff_memlists(phys_install, phys_avail, kphysm_erase); 2744 2745 build_pfn_hash(); 2746 } 2747 2748 /* 2749 * Kernel VM initialization. 2750 * Assumptions about kernel address space ordering: 2751 * (1) gap (user space) 2752 * (2) kernel text 2753 * (3) kernel data/bss 2754 * (4) gap 2755 * (5) kernel data structures 2756 * (6) gap 2757 * (7) debugger (optional) 2758 * (8) monitor 2759 * (9) gap (possibly null) 2760 * (10) dvma 2761 * (11) devices 2762 */ 2763 static void 2764 kvm_init(void) 2765 { 2766 /* 2767 * Put the kernel segments in kernel address space. 2768 */ 2769 rw_enter(&kas.a_lock, RW_WRITER); 2770 as_avlinit(&kas); 2771 2772 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2773 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2774 (void) segkmem_create(&ktextseg); 2775 2776 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2777 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2778 (void) segkmem_create(&ktexthole); 2779 2780 (void) seg_attach(&kas, (caddr_t)valloc_base, 2781 (size_t)(econtig32 - valloc_base), &kvalloc); 2782 (void) segkmem_create(&kvalloc); 2783 2784 if (kmem64_base) { 2785 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2786 (size_t)(kmem64_end - kmem64_base), &kmem64); 2787 (void) segkmem_create(&kmem64); 2788 } 2789 2790 /* 2791 * We're about to map out /boot. This is the beginning of the 2792 * system resource management transition. We can no longer 2793 * call into /boot for I/O or memory allocations. 2794 */ 2795 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2796 (void) segkmem_create(&kvseg); 2797 hblk_alloc_dynamic = 1; 2798 2799 /* 2800 * we need to preallocate pages for DR operations before enabling large 2801 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2802 */ 2803 memseg_remap_init(); 2804 2805 /* at this point we are ready to use large page heap */ 2806 segkmem_heap_lp_init(); 2807 2808 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2809 &kvseg32); 2810 (void) segkmem_create(&kvseg32); 2811 2812 /* 2813 * Create a segment for the debugger. 2814 */ 2815 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2816 (void) segkmem_create(&kdebugseg); 2817 2818 rw_exit(&kas.a_lock); 2819 } 2820 2821 char obp_tte_str[] = 2822 "h# %x constant MMU_PAGESHIFT " 2823 "h# %x constant TTE8K " 2824 "h# %x constant SFHME_SIZE " 2825 "h# %x constant SFHME_TTE " 2826 "h# %x constant HMEBLK_TAG " 2827 "h# %x constant HMEBLK_NEXT " 2828 "h# %x constant HMEBLK_MISC " 2829 "h# %x constant HMEBLK_HME1 " 2830 "h# %x constant NHMENTS " 2831 "h# %x constant HBLK_SZMASK " 2832 "h# %x constant HBLK_RANGE_SHIFT " 2833 "h# %x constant HMEBP_HBLK " 2834 "h# %x constant HMEBLK_ENDPA " 2835 "h# %x constant HMEBUCKET_SIZE " 2836 "h# %x constant HTAG_SFMMUPSZ " 2837 "h# %x constant HTAG_BSPAGE_SHIFT " 2838 "h# %x constant HTAG_REHASH_SHIFT " 2839 "h# %x constant SFMMU_INVALID_SHMERID " 2840 "h# %x constant mmu_hashcnt " 2841 "h# %p constant uhme_hash " 2842 "h# %p constant khme_hash " 2843 "h# %x constant UHMEHASH_SZ " 2844 "h# %x constant KHMEHASH_SZ " 2845 "h# %p constant KCONTEXT " 2846 "h# %p constant KHATID " 2847 "h# %x constant ASI_MEM " 2848 2849 ": PHYS-X@ ( phys -- data ) " 2850 " ASI_MEM spacex@ " 2851 "; " 2852 2853 ": PHYS-W@ ( phys -- data ) " 2854 " ASI_MEM spacew@ " 2855 "; " 2856 2857 ": PHYS-L@ ( phys -- data ) " 2858 " ASI_MEM spaceL@ " 2859 "; " 2860 2861 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2862 " 3 * MMU_PAGESHIFT + " 2863 "; " 2864 2865 ": TTE_IS_VALID ( ttep -- flag ) " 2866 " PHYS-X@ 0< " 2867 "; " 2868 2869 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2870 " dup TTE8K = if " 2871 " drop HBLK_RANGE_SHIFT " 2872 " else " 2873 " TTE_PAGE_SHIFT " 2874 " then " 2875 "; " 2876 2877 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2878 " tuck >> swap MMU_PAGESHIFT - << " 2879 "; " 2880 2881 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2882 " >> over xor swap ( hash sfmmup ) " 2883 " KHATID <> if ( hash ) " 2884 " UHMEHASH_SZ and ( bucket ) " 2885 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2886 " else ( hash ) " 2887 " KHMEHASH_SZ and ( bucket ) " 2888 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2889 " then ( hmebp ) " 2890 "; " 2891 2892 ": HME_HASH_TABLE_SEARCH " 2893 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2894 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2895 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2896 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2897 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2898 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2899 " else " 2900 " hmeblk_next + phys-x@ false " 2901 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2902 " then " 2903 " else " 2904 " hmeblk_next + phys-x@ false " 2905 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2906 " then " 2907 " else " 2908 " drop 0 true " 2909 " then " 2910 " until r> drop " 2911 "; " 2912 2913 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2914 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2915 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2916 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2917 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2918 "; " 2919 2920 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2921 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2922 " TTE8K = if ( hmeblkp addr ) " 2923 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2924 " else ( hmeblkp addr ) " 2925 " drop 0 ( hmeblkp 0 ) " 2926 " then ( hmeblkp hme-index ) " 2927 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2928 " SFHME_TTE + ( ttep ) " 2929 "; " 2930 2931 ": unix-tte ( addr cnum -- false | tte-data true ) " 2932 " KCONTEXT = if ( addr ) " 2933 " KHATID ( addr khatid ) " 2934 " else ( addr ) " 2935 " drop false exit ( false ) " 2936 " then " 2937 " ( addr khatid ) " 2938 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2939 " 2dup swap i HME_HASH_SHIFT " 2940 "( addr sfmmup sfmmup addr hmeshift ) " 2941 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2942 " over i 4 pick " 2943 "( addr sfmmup hmebp sfmmup rehash addr ) " 2944 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2945 " HME_HASH_TABLE_SEARCH " 2946 "( addr sfmmup { null | hmeblkp } ) " 2947 " ?dup if ( addr sfmmup hmeblkp ) " 2948 " nip swap HBLK_TO_TTEP ( ttep ) " 2949 " dup TTE_IS_VALID if ( valid-ttep ) " 2950 " PHYS-X@ true ( tte-data true ) " 2951 " else ( invalid-tte ) " 2952 " drop false ( false ) " 2953 " then ( false | tte-data true ) " 2954 " unloop exit ( false | tte-data true ) " 2955 " then ( addr sfmmup ) " 2956 " loop ( addr sfmmup ) " 2957 " 2drop false ( false ) " 2958 "; " 2959 ; 2960 2961 void 2962 create_va_to_tte(void) 2963 { 2964 char *bp; 2965 extern int khmehash_num, uhmehash_num; 2966 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2967 2968 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2969 2970 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2971 2972 /* 2973 * Teach obp how to parse our sw ttes. 2974 */ 2975 (void) sprintf(bp, obp_tte_str, 2976 MMU_PAGESHIFT, 2977 TTE8K, 2978 sizeof (struct sf_hment), 2979 OFFSET(struct sf_hment, hme_tte), 2980 OFFSET(struct hme_blk, hblk_tag), 2981 OFFSET(struct hme_blk, hblk_nextpa), 2982 OFFSET(struct hme_blk, hblk_misc), 2983 OFFSET(struct hme_blk, hblk_hme), 2984 NHMENTS, 2985 HBLK_SZMASK, 2986 HBLK_RANGE_SHIFT, 2987 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2988 HMEBLK_ENDPA, 2989 sizeof (struct hmehash_bucket), 2990 HTAG_SFMMUPSZ, 2991 HTAG_BSPAGE_SHIFT, 2992 HTAG_REHASH_SHIFT, 2993 SFMMU_INVALID_SHMERID, 2994 mmu_hashcnt, 2995 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2996 (caddr_t)va_to_pa((caddr_t)khme_hash), 2997 UHMEHASH_SZ, 2998 KHMEHASH_SZ, 2999 KCONTEXT, 3000 KHATID, 3001 ASI_MEM); 3002 prom_interpret(bp, 0, 0, 0, 0, 0); 3003 3004 kobj_free(bp, MMU_PAGESIZE); 3005 } 3006 3007 void 3008 install_va_to_tte(void) 3009 { 3010 /* 3011 * advise prom that he can use unix-tte 3012 */ 3013 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3014 } 3015 3016 /* 3017 * Here we add "device-type=console" for /os-io node, for currently 3018 * our kernel console output only supports displaying text and 3019 * performing cursor-positioning operations (through kernel framebuffer 3020 * driver) and it doesn't support other functionalities required for a 3021 * standard "display" device as specified in 1275 spec. The main missing 3022 * interface defined by the 1275 spec is "draw-logo". 3023 * also see the comments above prom_stdout_is_framebuffer(). 3024 */ 3025 static char *create_node = 3026 "\" /\" find-device " 3027 "new-device " 3028 "\" os-io\" device-name " 3029 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3030 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3031 " 2>r swap 2 2r> ['] $callback catch if " 3032 " 2drop 3drop 0 " 3033 " then " 3034 "; " 3035 ": read ( adr,len -- #read ) " 3036 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3037 " ( retN ... ret1 N ) " 3038 " ?dup if " 3039 " swap >r 1- 0 ?do drop loop r> " 3040 " else " 3041 " -2 " 3042 " then " 3043 "; " 3044 ": write ( adr,len -- #written ) " 3045 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3046 " ( retN ... ret1 N ) " 3047 " ?dup if " 3048 " swap >r 1- 0 ?do drop loop r> " 3049 " else " 3050 " 0 " 3051 " then " 3052 "; " 3053 ": poll-tty ( -- ) ; " 3054 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3055 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3056 ": cb-give/take ( $method -- ) " 3057 " 0 -rot ['] $callback catch ?dup if " 3058 " >r 2drop 2drop r> throw " 3059 " else " 3060 " 0 ?do drop loop " 3061 " then " 3062 "; " 3063 ": give ( -- ) \" exit-input\" cb-give/take ; " 3064 ": take ( -- ) \" enter-input\" cb-give/take ; " 3065 ": open ( -- ok? ) true ; " 3066 ": close ( -- ) ; " 3067 "finish-device " 3068 "device-end "; 3069 3070 /* 3071 * Create the OBP input/output node (FCode serial driver). 3072 * It is needed for both USB console keyboard and for 3073 * the kernel terminal emulator. It is too early to check for a 3074 * kernel console compatible framebuffer now, so we create this 3075 * so that we're ready if we need to enable kernel terminal emulation. 3076 * 3077 * When the USB software takes over the input device at the time 3078 * consconfig runs, OBP's stdin is redirected to this node. 3079 * Whenever the FORTH user interface is used after this switch, 3080 * the node will call back into the kernel for console input. 3081 * If a serial device such as ttya or a UART with a Type 5 keyboard 3082 * attached is used, OBP takes over the serial device when the system 3083 * goes to the debugger after the system is booted. This sharing 3084 * of the relatively simple serial device is difficult but possible. 3085 * Sharing the USB host controller is impossible due its complexity. 3086 * 3087 * Similarly to USB keyboard input redirection, after consconfig_dacf 3088 * configures a kernel console framebuffer as the standard output 3089 * device, OBP's stdout is switched to to vector through the 3090 * /os-io node into the kernel terminal emulator. 3091 */ 3092 static void 3093 startup_create_io_node(void) 3094 { 3095 prom_interpret(create_node, 0, 0, 0, 0, 0); 3096 } 3097 3098 3099 static void 3100 do_prom_version_check(void) 3101 { 3102 int i; 3103 pnode_t node; 3104 char buf[64]; 3105 static char drev[] = "Down-rev firmware detected%s\n" 3106 "\tPlease upgrade to the following minimum version:\n" 3107 "\t\t%s\n"; 3108 3109 i = prom_version_check(buf, sizeof (buf), &node); 3110 3111 if (i == PROM_VER64_OK) 3112 return; 3113 3114 if (i == PROM_VER64_UPGRADE) { 3115 cmn_err(CE_WARN, drev, "", buf); 3116 3117 #ifdef DEBUG 3118 prom_enter_mon(); /* Type 'go' to continue */ 3119 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3120 return; 3121 #else 3122 halt(0); 3123 #endif 3124 } 3125 3126 /* 3127 * The other possibility is that this is a server running 3128 * good firmware, but down-rev firmware was detected on at 3129 * least one other cpu board. We just complain if we see 3130 * that. 3131 */ 3132 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3133 } 3134 3135 3136 /* 3137 * Must be defined in platform dependent code. 3138 */ 3139 extern caddr_t modtext; 3140 extern size_t modtext_sz; 3141 extern caddr_t moddata; 3142 3143 #define HEAPTEXT_ARENA(addr) \ 3144 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3145 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3146 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3147 3148 #define HEAPTEXT_OVERSIZED(addr) \ 3149 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3150 3151 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3152 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3153 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3154 3155 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3156 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3157 kmutex_t texthole_lock; 3158 3159 char kern_bootargs[OBP_MAXPATHLEN]; 3160 char kern_bootfile[OBP_MAXPATHLEN]; 3161 3162 void 3163 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3164 { 3165 uintptr_t addr, limit; 3166 3167 addr = HEAPTEXT_BASE; 3168 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3169 3170 /* 3171 * Before we initialize the text_arena, we want to punch holes in the 3172 * underlying heaptext_arena. This guarantees that for any text 3173 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3174 */ 3175 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3176 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3177 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3178 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3179 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3180 } 3181 3182 /* 3183 * Allocate one page at the oversize to break up the text region 3184 * from the oversized region. 3185 */ 3186 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3187 (void *)limit, (void *)(limit + PAGESIZE), 3188 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3189 3190 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3191 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3192 heaptext_arena, 0, VM_SLEEP); 3193 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3194 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3195 } 3196 3197 caddr_t 3198 kobj_text_alloc(vmem_t *arena, size_t size) 3199 { 3200 caddr_t rval, better; 3201 3202 /* 3203 * First, try a sleeping allocation. 3204 */ 3205 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3206 3207 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3208 return (rval); 3209 3210 /* 3211 * We didn't get the area that we wanted. We're going to try to do an 3212 * allocation with explicit constraints. 3213 */ 3214 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3215 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3216 VM_NOSLEEP | VM_BESTFIT); 3217 3218 if (better != NULL) { 3219 /* 3220 * That worked. Free our first attempt and return. 3221 */ 3222 vmem_free(arena, rval, size); 3223 return (better); 3224 } 3225 3226 /* 3227 * That didn't work; we'll have to return our first attempt. 3228 */ 3229 return (rval); 3230 } 3231 3232 caddr_t 3233 kobj_texthole_alloc(caddr_t addr, size_t size) 3234 { 3235 int arena = HEAPTEXT_ARENA(addr); 3236 char c[30]; 3237 uintptr_t base; 3238 3239 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3240 /* 3241 * If this is an oversized allocation or it is allocated in 3242 * the nucleus data page, there is no text hole available for 3243 * it; return NULL. 3244 */ 3245 return (NULL); 3246 } 3247 3248 mutex_enter(&texthole_lock); 3249 3250 if (texthole_arena[arena] == NULL) { 3251 ASSERT(texthole_source[arena] == NULL); 3252 3253 if (arena == 0) { 3254 texthole_source[0] = vmem_create("module_text_holesrc", 3255 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3256 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3257 0, VM_SLEEP); 3258 } else { 3259 base = HEAPTEXT_BASE + 3260 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3261 3262 (void) snprintf(c, sizeof (c), 3263 "heaptext_holesrc_%d", arena); 3264 3265 texthole_source[arena] = vmem_create(c, (void *)base, 3266 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3267 0, VM_SLEEP); 3268 } 3269 3270 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3271 3272 texthole_arena[arena] = vmem_create(c, NULL, 0, 3273 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3274 texthole_source[arena], 0, VM_SLEEP); 3275 } 3276 3277 mutex_exit(&texthole_lock); 3278 3279 ASSERT(texthole_arena[arena] != NULL); 3280 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3281 return (vmem_alloc(texthole_arena[arena], size, 3282 VM_BESTFIT | VM_NOSLEEP)); 3283 } 3284 3285 void 3286 kobj_texthole_free(caddr_t addr, size_t size) 3287 { 3288 int arena = HEAPTEXT_ARENA(addr); 3289 3290 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3291 ASSERT(texthole_arena[arena] != NULL); 3292 vmem_free(texthole_arena[arena], addr, size); 3293 } 3294 3295 void 3296 release_bootstrap(void) 3297 { 3298 if (&cif_init) 3299 cif_init(); 3300 } 3301