1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/machsystm.h> 28 #include <sys/archsystm.h> 29 #include <sys/vm.h> 30 #include <sys/cpu.h> 31 #include <sys/atomic.h> 32 #include <sys/reboot.h> 33 #include <sys/kdi.h> 34 #include <sys/bootconf.h> 35 #include <sys/memlist_plat.h> 36 #include <sys/memlist_impl.h> 37 #include <sys/prom_plat.h> 38 #include <sys/prom_isa.h> 39 #include <sys/autoconf.h> 40 #include <sys/ivintr.h> 41 #include <sys/fpu/fpusystm.h> 42 #include <sys/iommutsb.h> 43 #include <vm/vm_dep.h> 44 #include <vm/seg_dev.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_kpm.h> 47 #include <vm/seg_map.h> 48 #include <vm/seg_kp.h> 49 #include <sys/sysconf.h> 50 #include <vm/hat_sfmmu.h> 51 #include <sys/kobj.h> 52 #include <sys/sun4asi.h> 53 #include <sys/clconf.h> 54 #include <sys/platform_module.h> 55 #include <sys/panic.h> 56 #include <sys/cpu_sgnblk_defs.h> 57 #include <sys/clock.h> 58 #include <sys/cmn_err.h> 59 #include <sys/dumphdr.h> 60 #include <sys/promif.h> 61 #include <sys/prom_debug.h> 62 #include <sys/traptrace.h> 63 #include <sys/memnode.h> 64 #include <sys/mem_cage.h> 65 #include <sys/mmu.h> 66 #include <sys/swap.h> 67 68 extern void setup_trap_table(void); 69 extern int cpu_intrq_setup(struct cpu *); 70 extern void cpu_intrq_register(struct cpu *); 71 extern void contig_mem_init(void); 72 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 73 extern void mach_dump_buffer_init(void); 74 extern void mach_descrip_init(void); 75 extern void mach_descrip_startup_fini(void); 76 extern void mach_memscrub(void); 77 extern void mach_fpras(void); 78 extern void mach_cpu_halt_idle(void); 79 extern void mach_hw_copy_limit(void); 80 extern void load_mach_drivers(void); 81 extern void load_tod_module(void); 82 #pragma weak load_tod_module 83 84 extern int ndata_alloc_mmfsa(struct memlist *ndata); 85 #pragma weak ndata_alloc_mmfsa 86 87 extern void cif_init(void); 88 #pragma weak cif_init 89 90 extern void parse_idprom(void); 91 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 92 extern void mem_config_init(void); 93 extern void memseg_remap_init(void); 94 95 extern void mach_kpm_init(void); 96 extern void pcf_init(); 97 extern int size_pse_array(pgcnt_t, int); 98 extern void pg_init(); 99 100 /* 101 * External Data: 102 */ 103 extern int vac_size; /* cache size in bytes */ 104 extern uint_t vac_mask; /* VAC alignment consistency mask */ 105 extern uint_t vac_colors; 106 107 /* 108 * Global Data Definitions: 109 */ 110 111 /* 112 * XXX - Don't port this to new architectures 113 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 114 * 'romp' has no use with a prom with an IEEE 1275 client interface. 115 * The driver doesn't use the value, but it depends on the symbol. 116 */ 117 void *romp; /* veritas driver won't load without romp 4154976 */ 118 /* 119 * Declare these as initialized data so we can patch them. 120 */ 121 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 122 pgcnt_t segkpsize = 123 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 124 uint_t segmap_percent = 6; /* Size of segmap segment */ 125 126 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 127 int vac_copyback = 1; 128 char *cache_mode = NULL; 129 int use_mix = 1; 130 int prom_debug = 0; 131 132 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 133 uint_t tba_taken_over = 0; 134 135 caddr_t s_text; /* start of kernel text segment */ 136 caddr_t e_text; /* end of kernel text segment */ 137 caddr_t s_data; /* start of kernel data segment */ 138 caddr_t e_data; /* end of kernel data segment */ 139 140 caddr_t modtext; /* beginning of module text */ 141 size_t modtext_sz; /* size of module text */ 142 caddr_t moddata; /* beginning of module data reserve */ 143 caddr_t e_moddata; /* end of module data reserve */ 144 145 /* 146 * End of first block of contiguous kernel in 32-bit virtual address space 147 */ 148 caddr_t econtig32; /* end of first blk of contiguous kernel */ 149 150 caddr_t ncbase; /* beginning of non-cached segment */ 151 caddr_t ncend; /* end of non-cached segment */ 152 153 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 154 caddr_t nalloc_base; /* beginning of nucleus allocation */ 155 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 156 caddr_t valloc_base; /* beginning of kvalloc segment */ 157 158 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 159 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 160 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 161 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 162 int kmem64_szc; /* page size code */ 163 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 164 165 uintptr_t shm_alignment; /* VAC address consistency modulus */ 166 struct memlist *phys_install; /* Total installed physical memory */ 167 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 168 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 169 struct memlist *nopp_list; /* pages with no backing page structs */ 170 struct memlist ndata; /* memlist of nucleus allocatable memory */ 171 int memexp_flag; /* memory expansion card flag */ 172 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 173 pgcnt_t obp_pages; /* Physical pages used by OBP */ 174 175 /* 176 * VM data structures 177 */ 178 long page_hashsz; /* Size of page hash table (power of two) */ 179 unsigned int page_hashsz_shift; /* log2(page_hashsz) */ 180 struct page *pp_base; /* Base of system page struct array */ 181 size_t pp_sz; /* Size in bytes of page struct array */ 182 struct page **page_hash; /* Page hash table */ 183 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 184 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 185 int pse_shift; /* log2(pse_table_size) */ 186 struct seg ktextseg; /* Segment used for kernel executable image */ 187 struct seg kvalloc; /* Segment used for "valloc" mapping */ 188 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 189 struct seg ktexthole; /* Segment used for nucleus text hole */ 190 struct seg kmapseg; /* Segment used for generic kernel mappings */ 191 struct seg kpmseg; /* Segment used for physical mapping */ 192 struct seg kdebugseg; /* Segment used for the kernel debugger */ 193 194 void *kpm_pp_base; /* Base of system kpm_page array */ 195 size_t kpm_pp_sz; /* Size of system kpm_page array */ 196 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 197 198 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 199 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 200 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 201 202 int segzio_fromheap = 0; /* zio allocations occur from heap */ 203 caddr_t segzio_base; /* Base address of segzio */ 204 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 205 206 /* 207 * A static DR page_t VA map is reserved that can map the page structures 208 * for a domain's entire RA space. The pages that backs this space are 209 * dynamically allocated and need not be physically contiguous. The DR 210 * map size is derived from KPM size. 211 */ 212 int ppvm_enable = 0; /* Static virtual map for page structs */ 213 page_t *ppvm_base; /* Base of page struct map */ 214 pgcnt_t ppvm_size = 0; /* Size of page struct map */ 215 216 /* 217 * debugger pages (if allocated) 218 */ 219 struct vnode kdebugvp; 220 221 /* 222 * VA range available to the debugger 223 */ 224 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 225 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 226 227 /* 228 * Segment for relocated kernel structures in 64-bit large RAM kernels 229 */ 230 struct seg kmem64; 231 232 struct memseg *memseg_free; 233 234 struct vnode unused_pages_vp; 235 236 /* 237 * VM data structures allocated early during boot. 238 */ 239 size_t pagehash_sz; 240 uint64_t memlist_sz; 241 242 char tbr_wr_addr_inited = 0; 243 244 caddr_t mpo_heap32_buf = NULL; 245 size_t mpo_heap32_bufsz = 0; 246 247 /* 248 * Static Routines: 249 */ 250 static int ndata_alloc_memseg(struct memlist *, size_t); 251 static void memlist_new(uint64_t, uint64_t, struct memlist **); 252 static void memlist_add(uint64_t, uint64_t, 253 struct memlist **, struct memlist **); 254 static void kphysm_init(void); 255 static void kvm_init(void); 256 static void install_kmem64_tte(void); 257 258 static void startup_init(void); 259 static void startup_memlist(void); 260 static void startup_modules(void); 261 static void startup_bop_gone(void); 262 static void startup_vm(void); 263 static void startup_end(void); 264 static void setup_cage_params(void); 265 static void startup_create_io_node(void); 266 267 static pgcnt_t npages; 268 static struct memlist *memlist; 269 void *memlist_end; 270 271 static pgcnt_t bop_alloc_pages; 272 static caddr_t hblk_base; 273 uint_t hblk_alloc_dynamic = 0; 274 uint_t hblk1_min = H1MIN; 275 276 277 /* 278 * Hooks for unsupported platforms and down-rev firmware 279 */ 280 int iam_positron(void); 281 #pragma weak iam_positron 282 static void do_prom_version_check(void); 283 284 /* 285 * After receiving a thermal interrupt, this is the number of seconds 286 * to delay before shutting off the system, assuming 287 * shutdown fails. Use /etc/system to change the delay if this isn't 288 * large enough. 289 */ 290 int thermal_powerdown_delay = 1200; 291 292 /* 293 * Used to hold off page relocations into the cage until OBP has completed 294 * its boot-time handoff of its resources to the kernel. 295 */ 296 int page_relocate_ready = 0; 297 298 /* 299 * Indicate if kmem64 allocation was done in small chunks 300 */ 301 int kmem64_smchunks = 0; 302 303 /* 304 * Enable some debugging messages concerning memory usage... 305 */ 306 #ifdef DEBUGGING_MEM 307 static int debugging_mem; 308 static void 309 printmemlist(char *title, struct memlist *list) 310 { 311 if (!debugging_mem) 312 return; 313 314 printf("%s\n", title); 315 316 while (list) { 317 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 318 (uint32_t)(list->ml_address >> 32), 319 (uint32_t)list->ml_address, 320 (uint32_t)(list->ml_size >> 32), 321 (uint32_t)(list->ml_size)); 322 list = list->ml_next; 323 } 324 } 325 326 void 327 printmemseg(struct memseg *memseg) 328 { 329 if (!debugging_mem) 330 return; 331 332 printf("memseg\n"); 333 334 while (memseg) { 335 prom_printf("\tpage = 0x%p, epage = 0x%p, " 336 "pfn = 0x%x, epfn = 0x%x\n", 337 memseg->pages, memseg->epages, 338 memseg->pages_base, memseg->pages_end); 339 memseg = memseg->next; 340 } 341 } 342 343 #define debug_pause(str) halt((str)) 344 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 345 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 346 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 347 #define MPRINTF3(str, a, b, c) \ 348 if (debugging_mem) prom_printf((str), (a), (b), (c)) 349 #else /* DEBUGGING_MEM */ 350 #define MPRINTF(str) 351 #define MPRINTF1(str, a) 352 #define MPRINTF2(str, a, b) 353 #define MPRINTF3(str, a, b, c) 354 #endif /* DEBUGGING_MEM */ 355 356 357 /* 358 * 359 * Kernel's Virtual Memory Layout. 360 * /-----------------------\ 361 * 0xFFFFFFFF.FFFFFFFF -| |- 362 * | OBP's virtual page | 363 * | tables | 364 * 0xFFFFFFFC.00000000 -|-----------------------|- 365 * : : 366 * : : 367 * -|-----------------------|- 368 * | segzio | (base and size vary) 369 * 0xFFFFFE00.00000000 -|-----------------------|- 370 * | | Ultrasparc I/II support 371 * | segkpm segment | up to 2TB of physical 372 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 373 * | | 374 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 375 * : : 376 * : : 377 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 378 * | | ^ 379 * | UltraSPARC I/II call | | 380 * | bug requires an extra | | 381 * | 4 GB of space between | | 382 * | hole and used RAM | | 383 * | | | 384 * 0xFFFFF800.00000000 -|-----------------------|- | 385 * | | | 386 * | Virtual Address Hole | UltraSPARC 387 * | on UltraSPARC I/II | I/II * ONLY * 388 * | | | 389 * 0x00000800.00000000 -|-----------------------|- | 390 * | | | 391 * | UltraSPARC I/II call | | 392 * | bug requires an extra | | 393 * | 4 GB of space between | | 394 * | hole and used RAM | | 395 * | | v 396 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 397 * : : ^ 398 * : : | 399 * |-----------------------| | 400 * | | | 401 * | ecache flush area | | 402 * | (twice largest e$) | | 403 * | | | 404 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 405 * | overmapped area | alignend_end | 406 * | (kmem64_alignsize | | 407 * | boundary) | | 408 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 409 * | | | 410 * | 64-bit kernel ONLY | | 411 * | | | 412 * | kmem64 segment | | 413 * | | | 414 * | (Relocated extra HME | Approximately 415 * | block allocations, | 1 TB of virtual 416 * | memnode freelists, | address space 417 * | HME hash buckets, | | 418 * | mml_table, kpmp_table,| | 419 * | page_t array and | | 420 * | hashblock pool to | | 421 * | avoid hard-coded | | 422 * | 32-bit vaddr | | 423 * | limitations) | | 424 * | | v 425 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 426 * | | 427 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 428 * | | 429 * 0x00000300.00000000 -|-----------------------|- SYSBASE 430 * : : 431 * : : 432 * -|-----------------------|- 433 * | | 434 * | segmap segment | SEGMAPSIZE (1/8th physmem, 435 * | | 256G MAX) 436 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 437 * : : 438 * : : 439 * -|-----------------------|- 440 * | | 441 * | segkp | SEGKPSIZE (2GB) 442 * | | 443 * | | 444 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 445 * | | 446 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 447 * | | (SEGKPBASE - 0x400000) 448 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 449 * | | (MEMSCRUBBASE - NCARGS) 450 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 451 * | | (ARGSBASE - PPMAPSIZE) 452 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 453 * | | 454 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 455 * | | 456 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 457 * : : 458 * : : 459 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 460 * | | 461 * | OBP | 462 * | | 463 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 464 * | kmdb | 465 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 466 * : : 467 * : : 468 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 469 * | | 470 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 471 * | | ~64MB) 472 * -|-----------------------| 473 * | IVSIZE | 474 * 0x00000000.70004000 -|-----------------------| 475 * | panicbuf | 476 * 0x00000000.70002000 -|-----------------------| 477 * | PAGESIZE | 478 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 479 * | boot-time | 480 * | temporary space | 481 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 482 * : : 483 * : : 484 * | | 485 * |-----------------------|- econtig32 486 * | vm structures | 487 * 0x00000000.01C00000 |-----------------------|- nalloc_end 488 * | TSBs | 489 * |-----------------------|- end/nalloc_base 490 * | kernel data & bss | 491 * 0x00000000.01800000 -|-----------------------| 492 * : nucleus text hole : 493 * 0x00000000.01400000 -|-----------------------| 494 * : : 495 * |-----------------------| 496 * | module text | 497 * |-----------------------|- e_text/modtext 498 * | kernel text | 499 * |-----------------------| 500 * | trap table (48k) | 501 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 502 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 503 * |-----------------------| 504 * | | 505 * | invalid | 506 * | | 507 * 0x00000000.00000000 _|_______________________| 508 * 509 * 510 * 511 * 32-bit User Virtual Memory Layout. 512 * /-----------------------\ 513 * | | 514 * | invalid | 515 * | | 516 * 0xFFC00000 -|-----------------------|- USERLIMIT 517 * | user stack | 518 * : : 519 * : : 520 * : : 521 * | user data | 522 * -|-----------------------|- 523 * | user text | 524 * 0x00002000 -|-----------------------|- 525 * | invalid | 526 * 0x00000000 _|_______________________| 527 * 528 * 529 * 530 * 64-bit User Virtual Memory Layout. 531 * /-----------------------\ 532 * | | 533 * | invalid | 534 * | | 535 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 536 * | user stack | 537 * : : 538 * : : 539 * : : 540 * | user data | 541 * -|-----------------------|- 542 * | user text | 543 * 0x00000000.01000000 -|-----------------------|- 544 * | invalid | 545 * 0x00000000.00000000 _|_______________________| 546 */ 547 548 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 549 extern uint64_t ecache_flush_address(void); 550 551 #pragma weak load_platform_modules 552 #pragma weak plat_startup_memlist 553 #pragma weak ecache_init_scrub_flush_area 554 #pragma weak ecache_flush_address 555 556 557 /* 558 * By default the DR Cage is enabled for maximum OS 559 * MPSS performance. Users needing to disable the cage mechanism 560 * can set this variable to zero via /etc/system. 561 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 562 * will result in loss of DR functionality. 563 * Platforms wishing to disable kernel Cage by default 564 * should do so in their set_platform_defaults() routine. 565 */ 566 int kernel_cage_enable = 1; 567 568 static void 569 setup_cage_params(void) 570 { 571 void (*func)(void); 572 573 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 574 if (func != NULL) { 575 (*func)(); 576 return; 577 } 578 579 if (kernel_cage_enable == 0) { 580 return; 581 } 582 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 583 584 if (kcage_on) { 585 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 586 } else { 587 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 588 } 589 590 } 591 592 /* 593 * Machine-dependent startup code 594 */ 595 void 596 startup(void) 597 { 598 startup_init(); 599 if (&startup_platform) 600 startup_platform(); 601 startup_memlist(); 602 startup_modules(); 603 setup_cage_params(); 604 startup_bop_gone(); 605 startup_vm(); 606 startup_end(); 607 } 608 609 struct regs sync_reg_buf; 610 uint64_t sync_tt; 611 612 void 613 sync_handler(void) 614 { 615 struct panic_trap_info ti; 616 int i; 617 618 /* 619 * Prevent trying to talk to the other CPUs since they are 620 * sitting in the prom and won't reply. 621 */ 622 for (i = 0; i < NCPU; i++) { 623 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 624 cpu[i]->cpu_flags &= ~CPU_READY; 625 cpu[i]->cpu_flags |= CPU_QUIESCED; 626 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 627 } 628 } 629 630 /* 631 * Force a serial dump, since there are no CPUs to help. 632 */ 633 dump_plat_mincpu = 0; 634 635 /* 636 * We've managed to get here without going through the 637 * normal panic code path. Try and save some useful 638 * information. 639 */ 640 if (!panicstr && (curthread->t_panic_trap == NULL)) { 641 ti.trap_type = sync_tt; 642 ti.trap_regs = &sync_reg_buf; 643 ti.trap_addr = NULL; 644 ti.trap_mmu_fsr = 0x0; 645 646 curthread->t_panic_trap = &ti; 647 } 648 649 /* 650 * If we're re-entering the panic path, update the signature 651 * block so that the SC knows we're in the second part of panic. 652 */ 653 if (panicstr) 654 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 655 656 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 657 panic("sync initiated"); 658 } 659 660 661 static void 662 startup_init(void) 663 { 664 /* 665 * We want to save the registers while we're still in OBP 666 * so that we know they haven't been fiddled with since. 667 * (In principle, OBP can't change them just because it 668 * makes a callback, but we'd rather not depend on that 669 * behavior.) 670 */ 671 char sync_str[] = 672 "warning @ warning off : sync " 673 "%%tl-c %%tstate h# %p x! " 674 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 675 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 676 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 677 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 678 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 679 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 680 "%%y h# %p l! %%tl-c %%tt h# %p x! " 681 "sync ; warning !"; 682 683 /* 684 * 20 == num of %p substrings 685 * 16 == max num of chars %p will expand to. 686 */ 687 char bp[sizeof (sync_str) + 16 * 20]; 688 689 /* 690 * Initialize ptl1 stack for the 1st CPU. 691 */ 692 ptl1_init_cpu(&cpu0); 693 694 /* 695 * Initialize the address map for cache consistent mappings 696 * to random pages; must be done after vac_size is set. 697 */ 698 ppmapinit(); 699 700 /* 701 * Initialize the PROM callback handler. 702 */ 703 init_vx_handler(); 704 705 /* 706 * have prom call sync_callback() to handle the sync and 707 * save some useful information which will be stored in the 708 * core file later. 709 */ 710 (void) sprintf((char *)bp, sync_str, 711 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 712 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 713 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 714 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 715 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 716 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 717 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 718 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 719 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 720 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 721 prom_interpret(bp, 0, 0, 0, 0, 0); 722 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 723 } 724 725 726 size_t 727 calc_pp_sz(pgcnt_t npages) 728 { 729 730 return (npages * sizeof (struct page)); 731 } 732 733 size_t 734 calc_kpmpp_sz(pgcnt_t npages) 735 { 736 737 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 738 kpm_pgsz = 1ull << kpm_pgshft; 739 kpm_pgoff = kpm_pgsz - 1; 740 kpmp2pshft = kpm_pgshft - PAGESHIFT; 741 kpmpnpgs = 1 << kpmp2pshft; 742 743 if (kpm_smallpages == 0) { 744 /* 745 * Avoid fragmentation problems in kphysm_init() 746 * by allocating for all of physical memory 747 */ 748 kpm_npages = ptokpmpr(physinstalled); 749 return (kpm_npages * sizeof (kpm_page_t)); 750 } else { 751 kpm_npages = npages; 752 return (kpm_npages * sizeof (kpm_spage_t)); 753 } 754 } 755 756 size_t 757 calc_pagehash_sz(pgcnt_t npages) 758 { 759 /* LINTED */ 760 ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page)))); 761 /* 762 * The page structure hash table size is a power of 2 763 * such that the average hash chain length is PAGE_HASHAVELEN. 764 */ 765 page_hashsz = npages / PAGE_HASHAVELEN; 766 page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1), 767 highbit(page_hashsz)); 768 page_hashsz = 1 << page_hashsz_shift; 769 return (page_hashsz * sizeof (struct page *)); 770 } 771 772 int testkmem64_smchunks = 0; 773 774 int 775 alloc_kmem64(caddr_t base, caddr_t end) 776 { 777 int i; 778 caddr_t aligned_end = NULL; 779 780 if (testkmem64_smchunks) 781 return (1); 782 783 /* 784 * Make one large memory alloc after figuring out the 64-bit size. This 785 * will enable use of the largest page size appropriate for the system 786 * architecture. 787 */ 788 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 789 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 790 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 791 size_t alloc_size, alignsize; 792 #if !defined(C_OBP) 793 unsigned long long pa; 794 #endif /* !C_OBP */ 795 796 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 797 continue; 798 alignsize = TTEBYTES(i); 799 kmem64_szc = i; 800 801 /* limit page size for small memory */ 802 if (mmu_btop(alignsize) > (npages >> 2)) 803 continue; 804 805 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 806 alloc_size = aligned_end - base; 807 #if !defined(C_OBP) 808 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 809 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 810 kmem64_pabase = pa; 811 kmem64_aligned_end = aligned_end; 812 install_kmem64_tte(); 813 break; 814 } else { 815 prom_free_phys(alloc_size, pa); 816 } 817 } 818 #else /* !C_OBP */ 819 if (prom_alloc(base, alloc_size, alignsize) == base) { 820 kmem64_pabase = va_to_pa(kmem64_base); 821 kmem64_aligned_end = aligned_end; 822 break; 823 } 824 #endif /* !C_OBP */ 825 if (i == TTE8K) { 826 #ifdef sun4v 827 /* return failure to try small allocations */ 828 return (1); 829 #else 830 prom_panic("kmem64 allocation failure"); 831 #endif 832 } 833 } 834 ASSERT(aligned_end != NULL); 835 return (0); 836 } 837 838 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 839 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 840 841 #if !defined(C_OBP) 842 /* 843 * Install a temporary tte handler in OBP for kmem64 area. 844 * 845 * We map kmem64 area with large pages before the trap table is taken 846 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 847 * the same area. Duplicate tlb entries with different page sizes 848 * cause unpredicatble behavior. To avoid this, we don't create 849 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 850 * OBP). Instead, we manage translations with a temporary va>tte-data 851 * handler (kmem64-tte). This handler is replaced by unix-tte when 852 * the trap table is taken over. 853 * 854 * The temporary handler knows the physical address of the kmem64 855 * area. It uses the prom's pgmap@ Forth word for other addresses. 856 * 857 * We have to use BOP_ALLOC() method for C-OBP platforms because 858 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 859 * sun4u platforms. On sun4u we flush tlb after trap table is taken 860 * over if we use large pages for kernel heap and kmem64. Since sun4u 861 * prom (unlike sun4v) calls va>tte-data first for client address 862 * translation prom's ttes for kmem64 can't get into TLB even if we 863 * later switch to prom's trap table again. C-OBP uses 4M pages for 864 * client mappings when possible so on all platforms we get the 865 * benefit from large mappings for kmem64 area immediately during 866 * boot. 867 * 868 * pseudo code: 869 * if (context != 0) { 870 * return false 871 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 872 * tte = tte_template + 873 * (((miss_va & pagemask) - kmem64_base)); 874 * return tte, true 875 * } else { 876 * return pgmap@ result 877 * } 878 */ 879 char kmem64_obp_str[] = 880 "h# %lx constant kmem64-base " 881 "h# %lx constant kmem64-end " 882 "h# %lx constant kmem64-pagemask " 883 "h# %lx constant kmem64-template " 884 885 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 886 " if ( addr ) " 887 " drop false exit then ( false ) " 888 " dup kmem64-base kmem64-end within if ( addr ) " 889 " kmem64-pagemask and ( addr' ) " 890 " kmem64-base - ( addr' ) " 891 " kmem64-template + ( tte ) " 892 " true ( tte true ) " 893 " else ( addr ) " 894 " pgmap@ ( tte ) " 895 " dup 0< if true else drop false then ( tte true | false ) " 896 " then ( tte true | false ) " 897 "; " 898 899 "' kmem64-tte is va>tte-data " 900 ; 901 902 static void 903 install_kmem64_tte() 904 { 905 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 906 tte_t tte; 907 908 PRM_DEBUG(kmem64_pabase); 909 PRM_DEBUG(kmem64_szc); 910 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 911 PROC_DATA | HAT_NOSYNC, kmem64_szc); 912 PRM_DEBUG(tte.ll); 913 (void) sprintf(b, kmem64_obp_str, 914 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 915 ASSERT(strlen(b) < sizeof (b)); 916 prom_interpret(b, 0, 0, 0, 0, 0); 917 } 918 #endif /* !C_OBP */ 919 920 /* 921 * As OBP takes up some RAM when the system boots, pages will already be "lost" 922 * to the system and reflected in npages by the time we see it. 923 * 924 * We only want to allocate kernel structures in the 64-bit virtual address 925 * space on systems with enough RAM to make the overhead of keeping track of 926 * an extra kernel memory segment worthwhile. 927 * 928 * Since OBP has already performed its memory allocations by this point, if we 929 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 930 * memory in the 64-bit virtual address space; otherwise keep allocations 931 * contiguous with we've mapped so far in the 32-bit virtual address space. 932 */ 933 #define MINMOVE_RAM_MB ((size_t)1900) 934 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 935 #define BYTES_TO_MB(b) ((b) / 1048576ul) 936 937 pgcnt_t tune_npages = (pgcnt_t) 938 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 939 940 #pragma weak page_set_colorequiv_arr_cpu 941 extern void page_set_colorequiv_arr_cpu(void); 942 extern void page_set_colorequiv_arr(void); 943 944 static pgcnt_t ramdisk_npages; 945 static struct memlist *old_phys_avail; 946 947 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 948 949 static void 950 startup_memlist(void) 951 { 952 size_t hmehash_sz, pagelist_sz, tt_sz; 953 size_t psetable_sz; 954 caddr_t alloc_base; 955 caddr_t memspace; 956 struct memlist *cur; 957 size_t syslimit = (size_t)SYSLIMIT; 958 size_t sysbase = (size_t)SYSBASE; 959 960 /* 961 * Initialize enough of the system to allow kmem_alloc to work by 962 * calling boot to allocate its memory until the time that 963 * kvm_init is completed. The page structs are allocated after 964 * rounding up end to the nearest page boundary; the memsegs are 965 * initialized and the space they use comes from the kernel heap. 966 * With appropriate initialization, they can be reallocated later 967 * to a size appropriate for the machine's configuration. 968 * 969 * At this point, memory is allocated for things that will never 970 * need to be freed, this used to be "valloced". This allows a 971 * savings as the pages don't need page structures to describe 972 * them because them will not be managed by the vm system. 973 */ 974 975 /* 976 * We're loaded by boot with the following configuration (as 977 * specified in the sun4u/conf/Mapfile): 978 * 979 * text: 4 MB chunk aligned on a 4MB boundary 980 * data & bss: 4 MB chunk aligned on a 4MB boundary 981 * 982 * These two chunks will eventually be mapped by 2 locked 4MB 983 * ttes and will represent the nucleus of the kernel. This gives 984 * us some free space that is already allocated, some or all of 985 * which is made available to kernel module text. 986 * 987 * The free space in the data-bss chunk is used for nucleus 988 * allocatable data structures and we reserve it using the 989 * nalloc_base and nalloc_end variables. This space is currently 990 * being used for hat data structures required for tlb miss 991 * handling operations. We align nalloc_base to a l2 cache 992 * linesize because this is the line size the hardware uses to 993 * maintain cache coherency. 994 * 512K is carved out for module data. 995 */ 996 997 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 998 e_moddata = moddata + MODDATA; 999 nalloc_base = e_moddata; 1000 1001 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 1002 valloc_base = nalloc_base; 1003 1004 /* 1005 * Calculate the start of the data segment. 1006 */ 1007 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 1008 prom_panic("nucleus data overflow"); 1009 1010 PRM_DEBUG(moddata); 1011 PRM_DEBUG(nalloc_base); 1012 PRM_DEBUG(nalloc_end); 1013 1014 /* 1015 * Remember any slop after e_text so we can give it to the modules. 1016 */ 1017 PRM_DEBUG(e_text); 1018 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1019 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1020 prom_panic("nucleus text overflow"); 1021 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1022 modtext; 1023 PRM_DEBUG(modtext); 1024 PRM_DEBUG(modtext_sz); 1025 1026 init_boot_memlists(); 1027 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1028 &boot_physavail, &boot_physavail_len, 1029 &boot_virtavail, &boot_virtavail_len); 1030 1031 /* 1032 * Remember what the physically available highest page is 1033 * so that dumpsys works properly, and find out how much 1034 * memory is installed. 1035 */ 1036 installed_top_size_memlist_array(boot_physinstalled, 1037 boot_physinstalled_len, &physmax, &physinstalled); 1038 PRM_DEBUG(physinstalled); 1039 PRM_DEBUG(physmax); 1040 1041 /* Fill out memory nodes config structure */ 1042 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1043 1044 /* 1045 * npages is the maximum of available physical memory possible. 1046 * (ie. it will never be more than this) 1047 * 1048 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1049 * using phys_avail will underestimate what will end up being freed. 1050 * A better initial guess is just total memory minus the kernel text 1051 */ 1052 npages = physinstalled - btop(MMU_PAGESIZE4M); 1053 1054 /* 1055 * First allocate things that can go in the nucleus data page 1056 * (fault status, TSBs, dmv, CPUs) 1057 */ 1058 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1059 1060 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1061 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1062 1063 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1064 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1065 1066 if (ndata_alloc_dmv(&ndata) != 0) 1067 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1068 1069 if (ndata_alloc_page_mutexs(&ndata) != 0) 1070 cmn_err(CE_PANIC, 1071 "no more nucleus memory after page free lists alloc"); 1072 1073 if (ndata_alloc_hat(&ndata) != 0) 1074 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1075 1076 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1077 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1078 1079 /* 1080 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1081 * 1082 * There are comments all over the SFMMU code warning of dire 1083 * consequences if the TSBs are moved out of 32-bit space. This 1084 * is largely because the asm code uses "sethi %hi(addr)"-type 1085 * instructions which will not provide the expected result if the 1086 * address is a 64-bit one. 1087 * 1088 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1089 */ 1090 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1091 PRM_DEBUG(alloc_base); 1092 1093 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1094 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1095 PRM_DEBUG(alloc_base); 1096 1097 /* 1098 * Allocate IOMMU TSB array. We do this here so that the physical 1099 * memory gets deducted from the PROM's physical memory list. 1100 */ 1101 alloc_base = iommu_tsb_init(alloc_base); 1102 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1103 PRM_DEBUG(alloc_base); 1104 1105 /* 1106 * Allow for an early allocation of physically contiguous memory. 1107 */ 1108 alloc_base = contig_mem_prealloc(alloc_base, npages); 1109 1110 /* 1111 * Platforms like Starcat and OPL need special structures assigned in 1112 * 32-bit virtual address space because their probing routines execute 1113 * FCode, and FCode can't handle 64-bit virtual addresses... 1114 */ 1115 if (&plat_startup_memlist) { 1116 alloc_base = plat_startup_memlist(alloc_base); 1117 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1118 ecache_alignsize); 1119 PRM_DEBUG(alloc_base); 1120 } 1121 1122 /* 1123 * Save off where the contiguous allocations to date have ended 1124 * in econtig32. 1125 */ 1126 econtig32 = alloc_base; 1127 PRM_DEBUG(econtig32); 1128 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1129 cmn_err(CE_PANIC, "econtig32 too big"); 1130 1131 pp_sz = calc_pp_sz(npages); 1132 PRM_DEBUG(pp_sz); 1133 if (kpm_enable) { 1134 kpm_pp_sz = calc_kpmpp_sz(npages); 1135 PRM_DEBUG(kpm_pp_sz); 1136 } 1137 1138 hmehash_sz = calc_hmehash_sz(npages); 1139 PRM_DEBUG(hmehash_sz); 1140 1141 pagehash_sz = calc_pagehash_sz(npages); 1142 PRM_DEBUG(pagehash_sz); 1143 1144 pagelist_sz = calc_free_pagelist_sz(); 1145 PRM_DEBUG(pagelist_sz); 1146 1147 #ifdef TRAPTRACE 1148 tt_sz = calc_traptrace_sz(); 1149 PRM_DEBUG(tt_sz); 1150 #else 1151 tt_sz = 0; 1152 #endif /* TRAPTRACE */ 1153 1154 /* 1155 * Place the array that protects pp->p_selock in the kmem64 wad. 1156 */ 1157 pse_shift = size_pse_array(npages, max_ncpus); 1158 PRM_DEBUG(pse_shift); 1159 pse_table_size = 1 << pse_shift; 1160 PRM_DEBUG(pse_table_size); 1161 psetable_sz = roundup( 1162 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1163 PRM_DEBUG(psetable_sz); 1164 1165 /* 1166 * Now allocate the whole wad 1167 */ 1168 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1169 pagelist_sz + tt_sz + psetable_sz; 1170 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1171 kmem64_base = (caddr_t)syslimit; 1172 kmem64_end = kmem64_base + kmem64_sz; 1173 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1174 /* 1175 * Attempt for kmem64 to allocate one big 1176 * contiguous chunk of memory failed. 1177 * We get here because we are sun4v. 1178 * We will proceed by breaking up 1179 * the allocation into two attempts. 1180 * First, we allocate kpm_pp_sz, hmehash_sz, 1181 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1182 * one contiguous chunk. This is a much smaller 1183 * chunk and we should get it, if not we panic. 1184 * Note that hmehash and tt need to be physically 1185 * (in the real address sense) contiguous. 1186 * Next, we use bop_alloc_chunk() to 1187 * to allocate the page_t structures. 1188 * This will allow the page_t to be allocated 1189 * in multiple smaller chunks. 1190 * In doing so, the assumption that page_t is 1191 * physically contiguous no longer hold, this is ok 1192 * for sun4v but not for sun4u. 1193 */ 1194 size_t tmp_size; 1195 caddr_t tmp_base; 1196 1197 pp_sz = roundup(pp_sz, PAGESIZE); 1198 1199 /* 1200 * Allocate kpm_pp_sz, hmehash_sz, 1201 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1202 */ 1203 tmp_base = kmem64_base + pp_sz; 1204 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1205 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1206 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1207 prom_panic("kmem64 prom_alloc contig failed"); 1208 PRM_DEBUG(tmp_base); 1209 PRM_DEBUG(tmp_size); 1210 1211 /* 1212 * Allocate the page_ts 1213 */ 1214 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1215 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1216 PRM_DEBUG(kmem64_base); 1217 PRM_DEBUG(pp_sz); 1218 1219 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1220 ASSERT(kmem64_aligned_end >= kmem64_end); 1221 1222 kmem64_smchunks = 1; 1223 } else { 1224 1225 /* 1226 * We need to adjust pp_sz for the normal 1227 * case where kmem64 can allocate one large chunk 1228 */ 1229 if (kpm_smallpages == 0) { 1230 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1231 } else { 1232 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1233 sizeof (kpm_spage_t)); 1234 } 1235 pp_sz = npages * sizeof (struct page); 1236 } 1237 1238 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1239 cmn_err(CE_PANIC, "not enough kmem64 space"); 1240 PRM_DEBUG(kmem64_base); 1241 PRM_DEBUG(kmem64_end); 1242 PRM_DEBUG(kmem64_aligned_end); 1243 1244 /* 1245 * ... and divy it up 1246 */ 1247 alloc_base = kmem64_base; 1248 1249 pp_base = (page_t *)alloc_base; 1250 alloc_base += pp_sz; 1251 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1252 PRM_DEBUG(pp_base); 1253 PRM_DEBUG(npages); 1254 1255 if (kpm_enable) { 1256 kpm_pp_base = alloc_base; 1257 if (kpm_smallpages == 0) { 1258 /* kpm_npages based on physinstalled, don't reset */ 1259 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1260 } else { 1261 kpm_npages = ptokpmpr(npages); 1262 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1263 } 1264 alloc_base += kpm_pp_sz; 1265 alloc_base = 1266 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1267 PRM_DEBUG(kpm_pp_base); 1268 } 1269 1270 alloc_base = alloc_hmehash(alloc_base); 1271 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1272 PRM_DEBUG(alloc_base); 1273 1274 page_hash = (page_t **)alloc_base; 1275 alloc_base += pagehash_sz; 1276 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1277 PRM_DEBUG(page_hash); 1278 1279 alloc_base = alloc_page_freelists(alloc_base); 1280 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1281 PRM_DEBUG(alloc_base); 1282 1283 #ifdef TRAPTRACE 1284 ttrace_buf = alloc_base; 1285 alloc_base += tt_sz; 1286 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1287 PRM_DEBUG(alloc_base); 1288 #endif /* TRAPTRACE */ 1289 1290 pse_mutex = (pad_mutex_t *)alloc_base; 1291 alloc_base += psetable_sz; 1292 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1293 PRM_DEBUG(alloc_base); 1294 1295 /* 1296 * Note that if we use small chunk allocations for 1297 * kmem64, we need to ensure kmem64_end is the same as 1298 * kmem64_aligned_end to prevent subsequent logic from 1299 * trying to reuse the overmapping. 1300 * Otherwise we adjust kmem64_end to what we really allocated. 1301 */ 1302 if (kmem64_smchunks) { 1303 kmem64_end = kmem64_aligned_end; 1304 } else { 1305 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1306 } 1307 kmem64_sz = kmem64_end - kmem64_base; 1308 1309 if (&ecache_init_scrub_flush_area) { 1310 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1311 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1312 } 1313 1314 /* 1315 * If physmem is patched to be non-zero, use it instead of 1316 * the monitor value unless physmem is larger than the total 1317 * amount of memory on hand. 1318 */ 1319 if (physmem == 0 || physmem > npages) 1320 physmem = npages; 1321 1322 /* 1323 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1324 * is mounted as root. This memory is held down by OBP and unlike 1325 * the stub boot_archive is never released. 1326 * 1327 * In order to get things sized correctly on lower memory 1328 * machines (where the memory used by the ramdisk represents 1329 * a significant portion of memory), physmem is adjusted. 1330 * 1331 * This is done by subtracting the ramdisk_size which is set 1332 * to the size of the ramdisk (in Kb) in /etc/system at the 1333 * time the miniroot archive is constructed. 1334 */ 1335 if (root_is_ramdisk == B_TRUE) { 1336 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1337 physmem -= ramdisk_npages; 1338 } 1339 1340 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1341 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1342 1343 /* 1344 * Allocate space for the interrupt vector table. 1345 */ 1346 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1347 if (memspace != (caddr_t)intr_vec_table) 1348 prom_panic("interrupt vector table allocation failure"); 1349 1350 /* 1351 * Between now and when we finish copying in the memory lists, 1352 * allocations happen so the space gets fragmented and the 1353 * lists longer. Leave enough space for lists twice as 1354 * long as we have now; then roundup to a pagesize. 1355 */ 1356 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1357 prom_phys_avail_len() + prom_virt_avail_len()); 1358 memlist_sz *= 2; 1359 memlist_sz = roundup(memlist_sz, PAGESIZE); 1360 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1361 if (memspace == NULL) 1362 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1363 1364 memlist = (struct memlist *)memspace; 1365 memlist_end = (char *)memspace + memlist_sz; 1366 PRM_DEBUG(memlist); 1367 PRM_DEBUG(memlist_end); 1368 1369 PRM_DEBUG(sysbase); 1370 PRM_DEBUG(syslimit); 1371 kernelheap_init((void *)sysbase, (void *)syslimit, 1372 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1373 1374 /* 1375 * Take the most current snapshot we can by calling mem-update. 1376 */ 1377 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1378 &boot_physavail, &boot_physavail_len, 1379 &boot_virtavail, &boot_virtavail_len); 1380 1381 /* 1382 * Remove the space used by prom_alloc from the kernel heap 1383 * plus the area actually used by the OBP (if any) 1384 * ignoring virtual addresses in virt_avail, above syslimit. 1385 */ 1386 virt_avail = memlist; 1387 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1388 1389 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) { 1390 uint64_t range_base, range_size; 1391 1392 if ((range_base = cur->ml_address + cur->ml_size) < 1393 (uint64_t)sysbase) 1394 continue; 1395 if (range_base >= (uint64_t)syslimit) 1396 break; 1397 /* 1398 * Limit the range to end at syslimit. 1399 */ 1400 range_size = MIN(cur->ml_next->ml_address, 1401 (uint64_t)syslimit) - range_base; 1402 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1403 0, 0, (void *)range_base, (void *)(range_base + range_size), 1404 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1405 } 1406 1407 phys_avail = memlist; 1408 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1409 1410 /* 1411 * Add any extra memory at the end of the ndata region if there's at 1412 * least a page to add. There might be a few more pages available in 1413 * the middle of the ndata region, but for now they are ignored. 1414 */ 1415 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1416 if (nalloc_base == NULL) 1417 nalloc_base = nalloc_end; 1418 ndata_remain_sz = nalloc_end - nalloc_base; 1419 1420 /* 1421 * Copy physinstalled list into kernel space. 1422 */ 1423 phys_install = memlist; 1424 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1425 1426 /* 1427 * Create list of physical addrs we don't need pp's for: 1428 * kernel text 4M page 1429 * kernel data 4M page - ndata_remain_sz 1430 * kmem64 pages 1431 * 1432 * NB if adding any pages here, make sure no kpm page 1433 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1434 */ 1435 nopp_list = memlist; 1436 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1437 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1438 &memlist, &nopp_list); 1439 1440 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1441 if (!kmem64_smchunks) 1442 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1443 1444 if ((caddr_t)memlist > (memspace + memlist_sz)) 1445 prom_panic("memlist overflow"); 1446 1447 /* 1448 * Size the pcf array based on the number of cpus in the box at 1449 * boot time. 1450 */ 1451 pcf_init(); 1452 1453 /* 1454 * Initialize the page structures from the memory lists. 1455 */ 1456 kphysm_init(); 1457 1458 availrmem_initial = availrmem = freemem; 1459 PRM_DEBUG(availrmem); 1460 1461 /* 1462 * Some of the locks depend on page_hashsz being set! 1463 * kmem_init() depends on this; so, keep it here. 1464 */ 1465 page_lock_init(); 1466 1467 /* 1468 * Initialize kernel memory allocator. 1469 */ 1470 kmem_init(); 1471 1472 /* 1473 * Factor in colorequiv to check additional 'equivalent' bins 1474 */ 1475 if (&page_set_colorequiv_arr_cpu != NULL) 1476 page_set_colorequiv_arr_cpu(); 1477 else 1478 page_set_colorequiv_arr(); 1479 1480 /* 1481 * Initialize bp_mapin(). 1482 */ 1483 bp_init(shm_alignment, HAT_STRICTORDER); 1484 1485 /* 1486 * Reserve space for MPO mblock structs from the 32-bit heap. 1487 */ 1488 1489 if (mpo_heap32_bufsz > (size_t)0) { 1490 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1491 PAGESIZE, 0, 0, mpo_heap32_buf, 1492 mpo_heap32_buf + mpo_heap32_bufsz, 1493 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1494 } 1495 mem_config_init(); 1496 } 1497 1498 static void 1499 startup_modules(void) 1500 { 1501 int nhblk1, nhblk8; 1502 size_t nhblksz; 1503 pgcnt_t pages_per_hblk; 1504 size_t hme8blk_sz, hme1blk_sz; 1505 1506 /* 1507 * The system file /etc/system was read already under startup_memlist. 1508 */ 1509 if (&set_platform_defaults) 1510 set_platform_defaults(); 1511 1512 /* 1513 * Calculate default settings of system parameters based upon 1514 * maxusers, yet allow to be overridden via the /etc/system file. 1515 */ 1516 param_calc(0); 1517 1518 mod_setup(); 1519 1520 /* 1521 * If this is a positron, complain and halt. 1522 */ 1523 if (&iam_positron && iam_positron()) { 1524 cmn_err(CE_WARN, "This hardware platform is not supported" 1525 " by this release of Solaris.\n"); 1526 #ifdef DEBUG 1527 prom_enter_mon(); /* Type 'go' to resume */ 1528 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1529 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1530 1531 #else /* DEBUG */ 1532 halt(0); 1533 #endif /* DEBUG */ 1534 } 1535 1536 /* 1537 * If we are running firmware that isn't 64-bit ready 1538 * then complain and halt. 1539 */ 1540 do_prom_version_check(); 1541 1542 /* 1543 * Initialize system parameters 1544 */ 1545 param_init(); 1546 1547 /* 1548 * maxmem is the amount of physical memory we're playing with. 1549 */ 1550 maxmem = physmem; 1551 1552 /* Set segkp limits. */ 1553 ncbase = kdi_segdebugbase; 1554 ncend = kdi_segdebugbase; 1555 1556 /* 1557 * Initialize the hat layer. 1558 */ 1559 hat_init(); 1560 1561 /* 1562 * Initialize segment management stuff. 1563 */ 1564 seg_init(); 1565 1566 /* 1567 * Create the va>tte handler, so the prom can understand 1568 * kernel translations. The handler is installed later, just 1569 * as we are about to take over the trap table from the prom. 1570 */ 1571 create_va_to_tte(); 1572 1573 /* 1574 * Load the forthdebugger (optional) 1575 */ 1576 forthdebug_init(); 1577 1578 /* 1579 * Create OBP node for console input callbacks 1580 * if it is needed. 1581 */ 1582 startup_create_io_node(); 1583 1584 if (modloadonly("fs", "specfs") == -1) 1585 halt("Can't load specfs"); 1586 1587 if (modloadonly("fs", "devfs") == -1) 1588 halt("Can't load devfs"); 1589 1590 if (modloadonly("fs", "procfs") == -1) 1591 halt("Can't load procfs"); 1592 1593 if (modloadonly("misc", "swapgeneric") == -1) 1594 halt("Can't load swapgeneric"); 1595 1596 (void) modloadonly("sys", "lbl_edition"); 1597 1598 dispinit(); 1599 1600 /* 1601 * Infer meanings to the members of the idprom buffer. 1602 */ 1603 parse_idprom(); 1604 1605 /* Read cluster configuration data. */ 1606 clconf_init(); 1607 1608 setup_ddi(); 1609 1610 /* 1611 * Lets take this opportunity to load the root device. 1612 */ 1613 if (loadrootmodules() != 0) 1614 debug_enter("Can't load the root filesystem"); 1615 1616 /* 1617 * Load tod driver module for the tod part found on this system. 1618 * Recompute the cpu frequency/delays based on tod as tod part 1619 * tends to keep time more accurately. 1620 */ 1621 if (&load_tod_module) 1622 load_tod_module(); 1623 1624 /* 1625 * Allow platforms to load modules which might 1626 * be needed after bootops are gone. 1627 */ 1628 if (&load_platform_modules) 1629 load_platform_modules(); 1630 1631 setcpudelay(); 1632 1633 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1634 &boot_physavail, &boot_physavail_len, 1635 &boot_virtavail, &boot_virtavail_len); 1636 1637 /* 1638 * Calculation and allocation of hmeblks needed to remap 1639 * the memory allocated by PROM till now. 1640 * Overestimate the number of hblk1 elements by assuming 1641 * worst case of TTE64K mappings. 1642 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1643 */ 1644 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1645 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1646 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1647 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1648 1649 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1650 1651 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1652 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1653 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1654 1655 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1656 1657 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1658 nhblk8 = 0; 1659 while (bop_alloc_pages > 1) { 1660 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1661 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1662 bop_alloc_pages *= hme8blk_sz; 1663 bop_alloc_pages = btopr(bop_alloc_pages); 1664 } 1665 nhblk8 += 2; 1666 1667 /* 1668 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1669 * boundary, the number of hblk8's needed to map the entries in the 1670 * boot_virtavail list needs to be adjusted to take this into 1671 * consideration. Thus, we need to add additional hblk8's since it 1672 * is possible that an hblk8 will not have all 8 slots used due to 1673 * alignment constraints. Since there were boot_virtavail_len entries 1674 * in that list, we need to add that many hblk8's to the number 1675 * already calculated to make sure we don't underestimate. 1676 */ 1677 nhblk8 += boot_virtavail_len; 1678 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1679 1680 /* Allocate in pagesize chunks */ 1681 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1682 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1683 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1684 } 1685 1686 static void 1687 startup_bop_gone(void) 1688 { 1689 1690 /* 1691 * Destroy the MD initialized at startup 1692 * The startup initializes the MD framework 1693 * using prom and BOP alloc free it now. 1694 */ 1695 mach_descrip_startup_fini(); 1696 1697 /* 1698 * We're done with prom allocations. 1699 */ 1700 bop_fini(); 1701 1702 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1703 &boot_physavail, &boot_physavail_len, 1704 &boot_virtavail, &boot_virtavail_len); 1705 1706 /* 1707 * setup physically contiguous area twice as large as the ecache. 1708 * this is used while doing displacement flush of ecaches 1709 */ 1710 if (&ecache_flush_address) { 1711 ecache_flushaddr = ecache_flush_address(); 1712 if (ecache_flushaddr == (uint64_t)-1) { 1713 cmn_err(CE_PANIC, 1714 "startup: no memory to set ecache_flushaddr"); 1715 } 1716 } 1717 1718 /* 1719 * Virtual available next. 1720 */ 1721 ASSERT(virt_avail != NULL); 1722 memlist_free_list(virt_avail); 1723 virt_avail = memlist; 1724 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1725 1726 } 1727 1728 1729 /* 1730 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1731 * allocations have been performed. We can't call it in startup_bop_gone 1732 * since later operations can cause obp to allocate more memory. 1733 */ 1734 void 1735 startup_fixup_physavail(void) 1736 { 1737 struct memlist *cur; 1738 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1739 1740 PRM_DEBUG(kmem64_overmap_size); 1741 1742 /* 1743 * take the most current snapshot we can by calling mem-update 1744 */ 1745 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1746 &boot_physavail, &boot_physavail_len, 1747 &boot_virtavail, &boot_virtavail_len); 1748 1749 /* 1750 * Copy phys_avail list, again. 1751 * Both the kernel/boot and the prom have been allocating 1752 * from the original list we copied earlier. 1753 */ 1754 cur = memlist; 1755 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1756 1757 /* 1758 * Add any unused kmem64 memory from overmapped page 1759 * (Note: va_to_pa does not work for kmem64_end) 1760 */ 1761 if (kmem64_overmap_size) { 1762 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1763 kmem64_overmap_size, &memlist, &cur); 1764 } 1765 1766 /* 1767 * Add any extra memory after e_data we added to the phys_avail list 1768 * back to the old list. 1769 */ 1770 if (ndata_remain_sz >= MMU_PAGESIZE) 1771 memlist_add(va_to_pa(nalloc_base), 1772 (uint64_t)ndata_remain_sz, &memlist, &cur); 1773 1774 /* 1775 * There isn't any bounds checking on the memlist area 1776 * so ensure it hasn't overgrown. 1777 */ 1778 if ((caddr_t)memlist > (caddr_t)memlist_end) 1779 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1780 1781 /* 1782 * The kernel removes the pages that were allocated for it from 1783 * the freelist, but we now have to find any -extra- pages that 1784 * the prom has allocated for it's own book-keeping, and remove 1785 * them from the freelist too. sigh. 1786 */ 1787 sync_memlists(phys_avail, cur); 1788 1789 ASSERT(phys_avail != NULL); 1790 1791 old_phys_avail = phys_avail; 1792 phys_avail = cur; 1793 } 1794 1795 void 1796 update_kcage_ranges(uint64_t addr, uint64_t len) 1797 { 1798 pfn_t base = btop(addr); 1799 pgcnt_t num = btop(len); 1800 int rv; 1801 1802 rv = kcage_range_add(base, num, kcage_startup_dir); 1803 1804 if (rv == ENOMEM) { 1805 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1806 (len == 0 ? 0 : BYTES_TO_MB(len))); 1807 } else if (rv != 0) { 1808 /* catch this in debug kernels */ 1809 ASSERT(0); 1810 1811 cmn_err(CE_WARN, "unexpected kcage_range_add" 1812 " return value %d", rv); 1813 } 1814 } 1815 1816 static void 1817 startup_vm(void) 1818 { 1819 size_t i; 1820 struct segmap_crargs a; 1821 struct segkpm_crargs b; 1822 1823 uint64_t avmem; 1824 caddr_t va; 1825 pgcnt_t max_phys_segkp; 1826 int mnode; 1827 1828 extern int use_brk_lpg, use_stk_lpg; 1829 1830 /* 1831 * get prom's mappings, create hments for them and switch 1832 * to the kernel context. 1833 */ 1834 hat_kern_setup(); 1835 1836 /* 1837 * Take over trap table 1838 */ 1839 setup_trap_table(); 1840 1841 /* 1842 * Install the va>tte handler, so that the prom can handle 1843 * misses and understand the kernel table layout in case 1844 * we need call into the prom. 1845 */ 1846 install_va_to_tte(); 1847 1848 /* 1849 * Set a flag to indicate that the tba has been taken over. 1850 */ 1851 tba_taken_over = 1; 1852 1853 /* initialize MMU primary context register */ 1854 mmu_init_kcontext(); 1855 1856 /* 1857 * The boot cpu can now take interrupts, x-calls, x-traps 1858 */ 1859 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1860 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1861 1862 /* 1863 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1864 */ 1865 tbr_wr_addr_inited = 1; 1866 1867 /* 1868 * Initialize VM system, and map kernel address space. 1869 */ 1870 kvm_init(); 1871 1872 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1873 if (kernel_cage_enable) { 1874 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1875 } 1876 memlist_free_list(old_phys_avail); 1877 1878 /* 1879 * If the following is true, someone has patched 1880 * phsymem to be less than the number of pages that 1881 * the system actually has. Remove pages until system 1882 * memory is limited to the requested amount. Since we 1883 * have allocated page structures for all pages, we 1884 * correct the amount of memory we want to remove 1885 * by the size of the memory used to hold page structures 1886 * for the non-used pages. 1887 */ 1888 if (physmem + ramdisk_npages < npages) { 1889 pgcnt_t diff, off; 1890 struct page *pp; 1891 struct seg kseg; 1892 1893 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1894 1895 off = 0; 1896 diff = npages - (physmem + ramdisk_npages); 1897 diff -= mmu_btopr(diff * sizeof (struct page)); 1898 kseg.s_as = &kas; 1899 while (diff--) { 1900 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1901 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1902 &kseg, (caddr_t)off); 1903 if (pp == NULL) 1904 cmn_err(CE_PANIC, "limited physmem too much!"); 1905 page_io_unlock(pp); 1906 page_downgrade(pp); 1907 availrmem--; 1908 off += MMU_PAGESIZE; 1909 } 1910 } 1911 1912 /* 1913 * When printing memory, show the total as physmem less 1914 * that stolen by a debugger. 1915 */ 1916 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1917 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1918 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1919 1920 avmem = (uint64_t)freemem << PAGESHIFT; 1921 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1922 1923 /* 1924 * For small memory systems disable automatic large pages. 1925 */ 1926 if (physmem < privm_lpg_min_physmem) { 1927 use_brk_lpg = 0; 1928 use_stk_lpg = 0; 1929 } 1930 1931 /* 1932 * Perform platform specific freelist processing 1933 */ 1934 if (&plat_freelist_process) { 1935 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1936 if (mem_node_config[mnode].exists) 1937 plat_freelist_process(mnode); 1938 } 1939 1940 /* 1941 * Initialize the segkp segment type. We position it 1942 * after the configured tables and buffers (whose end 1943 * is given by econtig) and before V_WKBASE_ADDR. 1944 * Also in this area is segkmap (size SEGMAPSIZE). 1945 */ 1946 1947 /* XXX - cache alignment? */ 1948 va = (caddr_t)SEGKPBASE; 1949 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1950 1951 max_phys_segkp = (physmem * 2); 1952 1953 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1954 segkpsize = btop(SEGKPDEFSIZE); 1955 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1956 "segkpsize has been reset to %ld pages", segkpsize); 1957 } 1958 1959 i = ptob(MIN(segkpsize, max_phys_segkp)); 1960 1961 rw_enter(&kas.a_lock, RW_WRITER); 1962 if (seg_attach(&kas, va, i, segkp) < 0) 1963 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1964 if (segkp_create(segkp) != 0) 1965 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1966 rw_exit(&kas.a_lock); 1967 1968 /* 1969 * kpm segment 1970 */ 1971 segmap_kpm = kpm_enable && 1972 segmap_kpm && PAGESIZE == MAXBSIZE; 1973 1974 if (kpm_enable) { 1975 rw_enter(&kas.a_lock, RW_WRITER); 1976 1977 /* 1978 * The segkpm virtual range range is larger than the 1979 * actual physical memory size and also covers gaps in 1980 * the physical address range for the following reasons: 1981 * . keep conversion between segkpm and physical addresses 1982 * simple, cheap and unambiguous. 1983 * . avoid extension/shrink of the the segkpm in case of DR. 1984 * . avoid complexity for handling of virtual addressed 1985 * caches, segkpm and the regular mapping scheme must be 1986 * kept in sync wrt. the virtual color of mapped pages. 1987 * Any accesses to virtual segkpm ranges not backed by 1988 * physical memory will fall through the memseg pfn hash 1989 * and will be handled in segkpm_fault. 1990 * Additional kpm_size spaces needed for vac alias prevention. 1991 */ 1992 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1993 segkpm) < 0) 1994 cmn_err(CE_PANIC, "cannot attach segkpm"); 1995 1996 b.prot = PROT_READ | PROT_WRITE; 1997 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1998 1999 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2000 panic("segkpm_create segkpm"); 2001 2002 rw_exit(&kas.a_lock); 2003 2004 mach_kpm_init(); 2005 } 2006 2007 va = kpm_vbase + (kpm_size * vac_colors); 2008 2009 if (!segzio_fromheap) { 2010 size_t size; 2011 size_t physmem_b = mmu_ptob(physmem); 2012 2013 /* size is in bytes, segziosize is in pages */ 2014 if (segziosize == 0) { 2015 size = physmem_b; 2016 } else { 2017 size = mmu_ptob(segziosize); 2018 } 2019 2020 if (size < SEGZIOMINSIZE) { 2021 size = SEGZIOMINSIZE; 2022 } else if (size > SEGZIOMAXSIZE) { 2023 size = SEGZIOMAXSIZE; 2024 /* 2025 * On 64-bit x86, we only have 2TB of KVA. This exists 2026 * for parity with x86. 2027 * 2028 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2029 * doesn't consume all of KVA. However, if we have a 2030 * system that has more thant 512gb of physical memory, 2031 * we can actually consume about half of the difference 2032 * between 512gb and the rest of the available physical 2033 * memory. 2034 */ 2035 if (physmem_b > SEGZIOMAXSIZE) { 2036 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2037 } 2038 } 2039 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2040 /* put the base of the ZIO segment after the kpm segment */ 2041 segzio_base = va; 2042 va += mmu_ptob(segziosize); 2043 PRM_DEBUG(segziosize); 2044 PRM_DEBUG(segzio_base); 2045 2046 /* 2047 * On some platforms, kvm_init is called after the kpm 2048 * sizes have been determined. On SPARC, kvm_init is called 2049 * before, so we have to attach the kzioseg after kvm is 2050 * initialized, otherwise we'll try to allocate from the boot 2051 * area since the kernel heap hasn't yet been configured. 2052 */ 2053 rw_enter(&kas.a_lock, RW_WRITER); 2054 2055 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2056 &kzioseg); 2057 (void) segkmem_zio_create(&kzioseg); 2058 2059 /* create zio area covering new segment */ 2060 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2061 2062 rw_exit(&kas.a_lock); 2063 } 2064 2065 if (ppvm_enable) { 2066 caddr_t ppvm_max; 2067 2068 /* 2069 * ppvm refers to the static VA space used to map 2070 * the page_t's for dynamically added memory. 2071 * 2072 * ppvm_base should not cross a potential VA hole. 2073 * 2074 * ppvm_size should be large enough to map the 2075 * page_t's needed to manage all of KPM range. 2076 */ 2077 ppvm_size = 2078 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2079 MMU_PAGESIZE); 2080 ppvm_max = (caddr_t)(0ull - ppvm_size); 2081 ppvm_base = (page_t *)va; 2082 2083 if ((caddr_t)ppvm_base <= hole_end) { 2084 cmn_err(CE_WARN, 2085 "Memory DR disabled: invalid DR map base: 0x%p\n", 2086 (void *)ppvm_base); 2087 ppvm_enable = 0; 2088 } else if ((caddr_t)ppvm_base > ppvm_max) { 2089 uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2090 2091 cmn_err(CE_WARN, 2092 "Memory DR disabled: insufficient DR map size:" 2093 " 0x%lx (needed 0x%lx)\n", 2094 ppvm_size - diff, ppvm_size); 2095 ppvm_enable = 0; 2096 } 2097 PRM_DEBUG(ppvm_size); 2098 PRM_DEBUG(ppvm_base); 2099 } 2100 2101 /* 2102 * Now create generic mapping segment. This mapping 2103 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2104 * virtual address is greater than the amount of free 2105 * memory that is available, then we trim back the 2106 * segment size to that amount 2107 */ 2108 va = (caddr_t)SEGMAPBASE; 2109 2110 /* 2111 * 1201049: segkmap base address must be MAXBSIZE aligned 2112 */ 2113 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2114 2115 /* 2116 * Set size of segmap to percentage of freemem at boot, 2117 * but stay within the allowable range 2118 * Note we take percentage before converting from pages 2119 * to bytes to avoid an overflow on 32-bit kernels. 2120 */ 2121 i = mmu_ptob((freemem * segmap_percent) / 100); 2122 2123 if (i < MINMAPSIZE) 2124 i = MINMAPSIZE; 2125 2126 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2127 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2128 2129 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2130 2131 rw_enter(&kas.a_lock, RW_WRITER); 2132 if (seg_attach(&kas, va, i, segkmap) < 0) 2133 cmn_err(CE_PANIC, "cannot attach segkmap"); 2134 2135 a.prot = PROT_READ | PROT_WRITE; 2136 a.shmsize = shm_alignment; 2137 a.nfreelist = 0; /* use segmap driver defaults */ 2138 2139 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2140 panic("segmap_create segkmap"); 2141 rw_exit(&kas.a_lock); 2142 2143 segdev_init(); 2144 } 2145 2146 static void 2147 startup_end(void) 2148 { 2149 if ((caddr_t)memlist > (caddr_t)memlist_end) 2150 panic("memlist overflow 2"); 2151 memlist_free_block((caddr_t)memlist, 2152 ((caddr_t)memlist_end - (caddr_t)memlist)); 2153 memlist = NULL; 2154 2155 /* enable page_relocation since OBP is now done */ 2156 page_relocate_ready = 1; 2157 2158 /* 2159 * Perform tasks that get done after most of the VM 2160 * initialization has been done but before the clock 2161 * and other devices get started. 2162 */ 2163 kern_setup1(); 2164 2165 /* 2166 * Perform CPC initialization for this CPU. 2167 */ 2168 kcpc_hw_init(); 2169 2170 /* 2171 * Intialize the VM arenas for allocating physically 2172 * contiguus memory chunk for interrupt queues snd 2173 * allocate/register boot cpu's queues, if any and 2174 * allocate dump buffer for sun4v systems to store 2175 * extra crash information during crash dump 2176 */ 2177 contig_mem_init(); 2178 mach_descrip_init(); 2179 2180 if (cpu_intrq_setup(CPU)) { 2181 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2182 } 2183 cpu_intrq_register(CPU); 2184 mach_htraptrace_setup(CPU->cpu_id); 2185 mach_htraptrace_configure(CPU->cpu_id); 2186 mach_dump_buffer_init(); 2187 2188 /* 2189 * Initialize interrupt related stuff 2190 */ 2191 cpu_intr_alloc(CPU, NINTR_THREADS); 2192 2193 (void) splzs(); /* allow hi clock ints but not zs */ 2194 2195 /* 2196 * Initialize errors. 2197 */ 2198 error_init(); 2199 2200 /* 2201 * Note that we may have already used kernel bcopy before this 2202 * point - but if you really care about this, adb the use_hw_* 2203 * variables to 0 before rebooting. 2204 */ 2205 mach_hw_copy_limit(); 2206 2207 /* 2208 * Install the "real" preemption guards before DDI services 2209 * are available. 2210 */ 2211 (void) prom_set_preprom(kern_preprom); 2212 (void) prom_set_postprom(kern_postprom); 2213 CPU->cpu_m.mutex_ready = 1; 2214 2215 /* 2216 * Initialize segnf (kernel support for non-faulting loads). 2217 */ 2218 segnf_init(); 2219 2220 /* 2221 * Configure the root devinfo node. 2222 */ 2223 configure(); /* set up devices */ 2224 mach_cpu_halt_idle(); 2225 } 2226 2227 2228 void 2229 post_startup(void) 2230 { 2231 #ifdef PTL1_PANIC_DEBUG 2232 extern void init_ptl1_thread(void); 2233 #endif /* PTL1_PANIC_DEBUG */ 2234 extern void abort_sequence_init(void); 2235 2236 /* 2237 * Set the system wide, processor-specific flags to be passed 2238 * to userland via the aux vector for performance hints and 2239 * instruction set extensions. 2240 */ 2241 bind_hwcap(); 2242 2243 /* 2244 * Startup memory scrubber (if any) 2245 */ 2246 mach_memscrub(); 2247 2248 /* 2249 * Allocate soft interrupt to handle abort sequence. 2250 */ 2251 abort_sequence_init(); 2252 2253 /* 2254 * Configure the rest of the system. 2255 * Perform forceloading tasks for /etc/system. 2256 */ 2257 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2258 /* 2259 * ON4.0: Force /proc module in until clock interrupt handle fixed 2260 * ON4.0: This must be fixed or restated in /etc/systems. 2261 */ 2262 (void) modload("fs", "procfs"); 2263 2264 /* load machine class specific drivers */ 2265 load_mach_drivers(); 2266 2267 /* load platform specific drivers */ 2268 if (&load_platform_drivers) 2269 load_platform_drivers(); 2270 2271 /* load vis simulation module, if we are running w/fpu off */ 2272 if (!fpu_exists) { 2273 if (modload("misc", "vis") == -1) 2274 halt("Can't load vis"); 2275 } 2276 2277 mach_fpras(); 2278 2279 maxmem = freemem; 2280 2281 pg_init(); 2282 2283 #ifdef PTL1_PANIC_DEBUG 2284 init_ptl1_thread(); 2285 #endif /* PTL1_PANIC_DEBUG */ 2286 } 2287 2288 #ifdef PTL1_PANIC_DEBUG 2289 int ptl1_panic_test = 0; 2290 int ptl1_panic_xc_one_test = 0; 2291 int ptl1_panic_xc_all_test = 0; 2292 int ptl1_panic_xt_one_test = 0; 2293 int ptl1_panic_xt_all_test = 0; 2294 kthread_id_t ptl1_thread_p = NULL; 2295 kcondvar_t ptl1_cv; 2296 kmutex_t ptl1_mutex; 2297 int ptl1_recurse_count_threshold = 0x40; 2298 int ptl1_recurse_trap_threshold = 0x3d; 2299 extern void ptl1_recurse(int, int); 2300 extern void ptl1_panic_xt(int, int); 2301 2302 /* 2303 * Called once per second by timeout() to wake up 2304 * the ptl1_panic thread to see if it should cause 2305 * a trap to the ptl1_panic() code. 2306 */ 2307 /* ARGSUSED */ 2308 static void 2309 ptl1_wakeup(void *arg) 2310 { 2311 mutex_enter(&ptl1_mutex); 2312 cv_signal(&ptl1_cv); 2313 mutex_exit(&ptl1_mutex); 2314 } 2315 2316 /* 2317 * ptl1_panic cross call function: 2318 * Needed because xc_one() and xc_some() can pass 2319 * 64 bit args but ptl1_recurse() expects ints. 2320 */ 2321 static void 2322 ptl1_panic_xc(void) 2323 { 2324 ptl1_recurse(ptl1_recurse_count_threshold, 2325 ptl1_recurse_trap_threshold); 2326 } 2327 2328 /* 2329 * The ptl1 thread waits for a global flag to be set 2330 * and uses the recurse thresholds to set the stack depth 2331 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2332 * or indirectly via the cross call and cross trap functions. 2333 * 2334 * This is useful testing stack overflows and normal 2335 * ptl1_panic() states with a know stack frame. 2336 * 2337 * ptl1_recurse() is an asm function in ptl1_panic.s that 2338 * sets the {In, Local, Out, and Global} registers to a 2339 * know state on the stack and just prior to causing a 2340 * test ptl1_panic trap. 2341 */ 2342 static void 2343 ptl1_thread(void) 2344 { 2345 mutex_enter(&ptl1_mutex); 2346 while (ptl1_thread_p) { 2347 cpuset_t other_cpus; 2348 int cpu_id; 2349 int my_cpu_id; 2350 int target_cpu_id; 2351 int target_found; 2352 2353 if (ptl1_panic_test) { 2354 ptl1_recurse(ptl1_recurse_count_threshold, 2355 ptl1_recurse_trap_threshold); 2356 } 2357 2358 /* 2359 * Find potential targets for x-call and x-trap, 2360 * if any exist while preempt is disabled we 2361 * start a ptl1_panic if requested via a 2362 * globals. 2363 */ 2364 kpreempt_disable(); 2365 my_cpu_id = CPU->cpu_id; 2366 other_cpus = cpu_ready_set; 2367 CPUSET_DEL(other_cpus, CPU->cpu_id); 2368 target_found = 0; 2369 if (!CPUSET_ISNULL(other_cpus)) { 2370 /* 2371 * Pick the first one 2372 */ 2373 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2374 if (cpu_id == my_cpu_id) 2375 continue; 2376 2377 if (CPU_XCALL_READY(cpu_id)) { 2378 target_cpu_id = cpu_id; 2379 target_found = 1; 2380 break; 2381 } 2382 } 2383 ASSERT(target_found); 2384 2385 if (ptl1_panic_xc_one_test) { 2386 xc_one(target_cpu_id, 2387 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2388 } 2389 if (ptl1_panic_xc_all_test) { 2390 xc_some(other_cpus, 2391 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2392 } 2393 if (ptl1_panic_xt_one_test) { 2394 xt_one(target_cpu_id, 2395 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2396 } 2397 if (ptl1_panic_xt_all_test) { 2398 xt_some(other_cpus, 2399 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2400 } 2401 } 2402 kpreempt_enable(); 2403 (void) timeout(ptl1_wakeup, NULL, hz); 2404 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2405 } 2406 mutex_exit(&ptl1_mutex); 2407 } 2408 2409 /* 2410 * Called during early startup to create the ptl1_thread 2411 */ 2412 void 2413 init_ptl1_thread(void) 2414 { 2415 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2416 &p0, TS_RUN, 0); 2417 } 2418 #endif /* PTL1_PANIC_DEBUG */ 2419 2420 2421 static void 2422 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2423 { 2424 struct memlist *new; 2425 2426 new = *memlistp; 2427 new->ml_address = start; 2428 new->ml_size = len; 2429 *memlistp = new + 1; 2430 } 2431 2432 /* 2433 * Add to a memory list. 2434 * start = start of new memory segment 2435 * len = length of new memory segment in bytes 2436 * memlistp = pointer to array of available memory segment structures 2437 * curmemlistp = memory list to which to add segment. 2438 */ 2439 static void 2440 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2441 struct memlist **curmemlistp) 2442 { 2443 struct memlist *new = *memlistp; 2444 2445 memlist_new(start, len, memlistp); 2446 memlist_insert(new, curmemlistp); 2447 } 2448 2449 static int 2450 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2451 { 2452 int nseg; 2453 size_t memseg_sz; 2454 struct memseg *msp; 2455 2456 /* 2457 * The memseg list is for the chunks of physical memory that 2458 * will be managed by the vm system. The number calculated is 2459 * a guess as boot may fragment it more when memory allocations 2460 * are made before kphysm_init(). 2461 */ 2462 memseg_sz = (avail + 10) * sizeof (struct memseg); 2463 memseg_sz = roundup(memseg_sz, PAGESIZE); 2464 nseg = memseg_sz / sizeof (struct memseg); 2465 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2466 if (msp == NULL) 2467 return (1); 2468 PRM_DEBUG(memseg_free); 2469 2470 while (nseg--) { 2471 msp->next = memseg_free; 2472 memseg_free = msp; 2473 msp++; 2474 } 2475 return (0); 2476 } 2477 2478 /* 2479 * In the case of architectures that support dynamic addition of 2480 * memory at run-time there are two cases where memsegs need to 2481 * be initialized and added to the memseg list. 2482 * 1) memsegs that are constructed at startup. 2483 * 2) memsegs that are constructed at run-time on 2484 * hot-plug capable architectures. 2485 * This code was originally part of the function kphysm_init(). 2486 */ 2487 2488 static void 2489 memseg_list_add(struct memseg *memsegp) 2490 { 2491 struct memseg **prev_memsegp; 2492 pgcnt_t num; 2493 2494 /* insert in memseg list, decreasing number of pages order */ 2495 2496 num = MSEG_NPAGES(memsegp); 2497 2498 for (prev_memsegp = &memsegs; *prev_memsegp; 2499 prev_memsegp = &((*prev_memsegp)->next)) { 2500 if (num > MSEG_NPAGES(*prev_memsegp)) 2501 break; 2502 } 2503 2504 memsegp->next = *prev_memsegp; 2505 *prev_memsegp = memsegp; 2506 2507 if (kpm_enable) { 2508 memsegp->nextpa = (memsegp->next) ? 2509 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2510 2511 if (prev_memsegp != &memsegs) { 2512 struct memseg *msp; 2513 msp = (struct memseg *)((caddr_t)prev_memsegp - 2514 offsetof(struct memseg, next)); 2515 msp->nextpa = va_to_pa(memsegp); 2516 } else { 2517 memsegspa = va_to_pa(memsegs); 2518 } 2519 } 2520 } 2521 2522 /* 2523 * PSM add_physmem_cb(). US-II and newer processors have some 2524 * flavor of the prefetch capability implemented. We exploit 2525 * this capability for optimum performance. 2526 */ 2527 #define PREFETCH_BYTES 64 2528 2529 void 2530 add_physmem_cb(page_t *pp, pfn_t pnum) 2531 { 2532 extern void prefetch_page_w(void *); 2533 2534 pp->p_pagenum = pnum; 2535 2536 /* 2537 * Prefetch one more page_t into E$. To prevent future 2538 * mishaps with the sizeof(page_t) changing on us, we 2539 * catch this on debug kernels if we can't bring in the 2540 * entire hpage with 2 PREFETCH_BYTES reads. See 2541 * also, sun4u/cpu/cpu_module.c 2542 */ 2543 /*LINTED*/ 2544 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2545 prefetch_page_w((char *)pp); 2546 } 2547 2548 /* 2549 * Find memseg with given pfn 2550 */ 2551 static struct memseg * 2552 memseg_find(pfn_t base, pfn_t *next) 2553 { 2554 struct memseg *seg; 2555 2556 if (next != NULL) 2557 *next = LONG_MAX; 2558 for (seg = memsegs; seg != NULL; seg = seg->next) { 2559 if (base >= seg->pages_base && base < seg->pages_end) 2560 return (seg); 2561 if (next != NULL && seg->pages_base > base && 2562 seg->pages_base < *next) 2563 *next = seg->pages_base; 2564 } 2565 return (NULL); 2566 } 2567 2568 /* 2569 * Put page allocated by OBP on prom_ppages 2570 */ 2571 static void 2572 kphysm_erase(uint64_t addr, uint64_t len) 2573 { 2574 struct page *pp; 2575 struct memseg *seg; 2576 pfn_t base = btop(addr), next; 2577 pgcnt_t num = btop(len); 2578 2579 while (num != 0) { 2580 pgcnt_t off, left; 2581 2582 seg = memseg_find(base, &next); 2583 if (seg == NULL) { 2584 if (next == LONG_MAX) 2585 break; 2586 left = MIN(next - base, num); 2587 base += left, num -= left; 2588 continue; 2589 } 2590 off = base - seg->pages_base; 2591 pp = seg->pages + off; 2592 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2593 while (num != left) { 2594 /* 2595 * init it, lock it, and hashin on prom_pages vp. 2596 * 2597 * Mark it as NONRELOC to let DR know the page 2598 * is locked long term, otherwise DR hangs when 2599 * trying to remove those pages. 2600 * 2601 * XXX vnode offsets on the prom_ppages vnode 2602 * are page numbers (gack) for >32 bit 2603 * physical memory machines. 2604 */ 2605 PP_SETNORELOC(pp); 2606 add_physmem_cb(pp, base); 2607 if (page_trylock(pp, SE_EXCL) == 0) 2608 cmn_err(CE_PANIC, "prom page locked"); 2609 (void) page_hashin(pp, &promvp, 2610 (offset_t)base, NULL); 2611 (void) page_pp_lock(pp, 0, 1); 2612 pp++, base++, num--; 2613 } 2614 } 2615 } 2616 2617 static page_t *ppnext; 2618 static pgcnt_t ppleft; 2619 2620 static void *kpm_ppnext; 2621 static pgcnt_t kpm_ppleft; 2622 2623 /* 2624 * Create a memseg 2625 */ 2626 static void 2627 kphysm_memseg(uint64_t addr, uint64_t len) 2628 { 2629 pfn_t base = btop(addr); 2630 pgcnt_t num = btop(len); 2631 struct memseg *seg; 2632 2633 seg = memseg_free; 2634 memseg_free = seg->next; 2635 ASSERT(seg != NULL); 2636 2637 seg->pages = ppnext; 2638 seg->epages = ppnext + num; 2639 seg->pages_base = base; 2640 seg->pages_end = base + num; 2641 ppnext += num; 2642 ppleft -= num; 2643 2644 if (kpm_enable) { 2645 pgcnt_t kpnum = ptokpmpr(num); 2646 2647 if (kpnum > kpm_ppleft) 2648 panic("kphysm_memseg: kpm_pp overflow"); 2649 seg->pagespa = va_to_pa(seg->pages); 2650 seg->epagespa = va_to_pa(seg->epages); 2651 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2652 seg->kpm_nkpmpgs = kpnum; 2653 /* 2654 * In the kpm_smallpage case, the kpm array 2655 * is 1-1 wrt the page array 2656 */ 2657 if (kpm_smallpages) { 2658 kpm_spage_t *kpm_pp = kpm_ppnext; 2659 2660 kpm_ppnext = kpm_pp + kpnum; 2661 seg->kpm_spages = kpm_pp; 2662 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2663 } else { 2664 kpm_page_t *kpm_pp = kpm_ppnext; 2665 2666 kpm_ppnext = kpm_pp + kpnum; 2667 seg->kpm_pages = kpm_pp; 2668 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2669 /* ASSERT no kpm overlaps */ 2670 ASSERT( 2671 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2672 ASSERT(memseg_find( 2673 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2674 } 2675 kpm_ppleft -= kpnum; 2676 } 2677 2678 memseg_list_add(seg); 2679 } 2680 2681 /* 2682 * Add range to free list 2683 */ 2684 void 2685 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2686 { 2687 struct page *pp; 2688 struct memseg *seg; 2689 pfn_t base = btop(addr); 2690 pgcnt_t num = btop(len); 2691 2692 seg = memseg_find(base, NULL); 2693 ASSERT(seg != NULL); 2694 pp = seg->pages + (base - seg->pages_base); 2695 2696 if (reclaim) { 2697 struct page *rpp = pp; 2698 struct page *lpp = pp + num; 2699 2700 /* 2701 * page should be locked on prom_ppages 2702 * unhash and unlock it 2703 */ 2704 while (rpp < lpp) { 2705 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp); 2706 ASSERT(PP_ISNORELOC(rpp)); 2707 PP_CLRNORELOC(rpp); 2708 page_pp_unlock(rpp, 0, 1); 2709 page_hashout(rpp, NULL); 2710 page_unlock(rpp); 2711 rpp++; 2712 } 2713 } 2714 2715 /* 2716 * add_physmem() initializes the PSM part of the page 2717 * struct by calling the PSM back with add_physmem_cb(). 2718 * In addition it coalesces pages into larger pages as 2719 * it initializes them. 2720 */ 2721 add_physmem(pp, num, base); 2722 } 2723 2724 /* 2725 * kphysm_init() tackles the problem of initializing physical memory. 2726 */ 2727 static void 2728 kphysm_init(void) 2729 { 2730 struct memlist *pmem; 2731 2732 ASSERT(page_hash != NULL && page_hashsz != 0); 2733 2734 ppnext = pp_base; 2735 ppleft = npages; 2736 kpm_ppnext = kpm_pp_base; 2737 kpm_ppleft = kpm_npages; 2738 2739 /* 2740 * installed pages not on nopp_memlist go in memseg list 2741 */ 2742 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2743 2744 /* 2745 * Free the avail list 2746 */ 2747 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next) 2748 kphysm_add(pmem->ml_address, pmem->ml_size, 0); 2749 2750 /* 2751 * Erase pages that aren't available 2752 */ 2753 diff_memlists(phys_install, phys_avail, kphysm_erase); 2754 2755 build_pfn_hash(); 2756 } 2757 2758 /* 2759 * Kernel VM initialization. 2760 * Assumptions about kernel address space ordering: 2761 * (1) gap (user space) 2762 * (2) kernel text 2763 * (3) kernel data/bss 2764 * (4) gap 2765 * (5) kernel data structures 2766 * (6) gap 2767 * (7) debugger (optional) 2768 * (8) monitor 2769 * (9) gap (possibly null) 2770 * (10) dvma 2771 * (11) devices 2772 */ 2773 static void 2774 kvm_init(void) 2775 { 2776 /* 2777 * Put the kernel segments in kernel address space. 2778 */ 2779 rw_enter(&kas.a_lock, RW_WRITER); 2780 as_avlinit(&kas); 2781 2782 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2783 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2784 (void) segkmem_create(&ktextseg); 2785 2786 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2787 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2788 (void) segkmem_create(&ktexthole); 2789 2790 (void) seg_attach(&kas, (caddr_t)valloc_base, 2791 (size_t)(econtig32 - valloc_base), &kvalloc); 2792 (void) segkmem_create(&kvalloc); 2793 2794 if (kmem64_base) { 2795 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2796 (size_t)(kmem64_end - kmem64_base), &kmem64); 2797 (void) segkmem_create(&kmem64); 2798 } 2799 2800 /* 2801 * We're about to map out /boot. This is the beginning of the 2802 * system resource management transition. We can no longer 2803 * call into /boot for I/O or memory allocations. 2804 */ 2805 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2806 (void) segkmem_create(&kvseg); 2807 hblk_alloc_dynamic = 1; 2808 2809 /* 2810 * we need to preallocate pages for DR operations before enabling large 2811 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2812 */ 2813 memseg_remap_init(); 2814 2815 /* at this point we are ready to use large page heap */ 2816 segkmem_heap_lp_init(); 2817 2818 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2819 &kvseg32); 2820 (void) segkmem_create(&kvseg32); 2821 2822 /* 2823 * Create a segment for the debugger. 2824 */ 2825 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2826 (void) segkmem_create(&kdebugseg); 2827 2828 rw_exit(&kas.a_lock); 2829 } 2830 2831 char obp_tte_str[] = 2832 "h# %x constant MMU_PAGESHIFT " 2833 "h# %x constant TTE8K " 2834 "h# %x constant SFHME_SIZE " 2835 "h# %x constant SFHME_TTE " 2836 "h# %x constant HMEBLK_TAG " 2837 "h# %x constant HMEBLK_NEXT " 2838 "h# %x constant HMEBLK_MISC " 2839 "h# %x constant HMEBLK_HME1 " 2840 "h# %x constant NHMENTS " 2841 "h# %x constant HBLK_SZMASK " 2842 "h# %x constant HBLK_RANGE_SHIFT " 2843 "h# %x constant HMEBP_HBLK " 2844 "h# %x constant HMEBLK_ENDPA " 2845 "h# %x constant HMEBUCKET_SIZE " 2846 "h# %x constant HTAG_SFMMUPSZ " 2847 "h# %x constant HTAG_BSPAGE_SHIFT " 2848 "h# %x constant HTAG_REHASH_SHIFT " 2849 "h# %x constant SFMMU_INVALID_SHMERID " 2850 "h# %x constant mmu_hashcnt " 2851 "h# %p constant uhme_hash " 2852 "h# %p constant khme_hash " 2853 "h# %x constant UHMEHASH_SZ " 2854 "h# %x constant KHMEHASH_SZ " 2855 "h# %p constant KCONTEXT " 2856 "h# %p constant KHATID " 2857 "h# %x constant ASI_MEM " 2858 2859 ": PHYS-X@ ( phys -- data ) " 2860 " ASI_MEM spacex@ " 2861 "; " 2862 2863 ": PHYS-W@ ( phys -- data ) " 2864 " ASI_MEM spacew@ " 2865 "; " 2866 2867 ": PHYS-L@ ( phys -- data ) " 2868 " ASI_MEM spaceL@ " 2869 "; " 2870 2871 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2872 " 3 * MMU_PAGESHIFT + " 2873 "; " 2874 2875 ": TTE_IS_VALID ( ttep -- flag ) " 2876 " PHYS-X@ 0< " 2877 "; " 2878 2879 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2880 " dup TTE8K = if " 2881 " drop HBLK_RANGE_SHIFT " 2882 " else " 2883 " TTE_PAGE_SHIFT " 2884 " then " 2885 "; " 2886 2887 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2888 " tuck >> swap MMU_PAGESHIFT - << " 2889 "; " 2890 2891 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2892 " >> over xor swap ( hash sfmmup ) " 2893 " KHATID <> if ( hash ) " 2894 " UHMEHASH_SZ and ( bucket ) " 2895 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2896 " else ( hash ) " 2897 " KHMEHASH_SZ and ( bucket ) " 2898 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2899 " then ( hmebp ) " 2900 "; " 2901 2902 ": HME_HASH_TABLE_SEARCH " 2903 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2904 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2905 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2906 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2907 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2908 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2909 " else " 2910 " hmeblk_next + phys-x@ false " 2911 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2912 " then " 2913 " else " 2914 " hmeblk_next + phys-x@ false " 2915 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2916 " then " 2917 " else " 2918 " drop 0 true " 2919 " then " 2920 " until r> drop " 2921 "; " 2922 2923 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2924 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2925 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2926 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2927 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2928 "; " 2929 2930 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2931 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2932 " TTE8K = if ( hmeblkp addr ) " 2933 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2934 " else ( hmeblkp addr ) " 2935 " drop 0 ( hmeblkp 0 ) " 2936 " then ( hmeblkp hme-index ) " 2937 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2938 " SFHME_TTE + ( ttep ) " 2939 "; " 2940 2941 ": unix-tte ( addr cnum -- false | tte-data true ) " 2942 " KCONTEXT = if ( addr ) " 2943 " KHATID ( addr khatid ) " 2944 " else ( addr ) " 2945 " drop false exit ( false ) " 2946 " then " 2947 " ( addr khatid ) " 2948 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2949 " 2dup swap i HME_HASH_SHIFT " 2950 "( addr sfmmup sfmmup addr hmeshift ) " 2951 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2952 " over i 4 pick " 2953 "( addr sfmmup hmebp sfmmup rehash addr ) " 2954 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2955 " HME_HASH_TABLE_SEARCH " 2956 "( addr sfmmup { null | hmeblkp } ) " 2957 " ?dup if ( addr sfmmup hmeblkp ) " 2958 " nip swap HBLK_TO_TTEP ( ttep ) " 2959 " dup TTE_IS_VALID if ( valid-ttep ) " 2960 " PHYS-X@ true ( tte-data true ) " 2961 " else ( invalid-tte ) " 2962 " drop false ( false ) " 2963 " then ( false | tte-data true ) " 2964 " unloop exit ( false | tte-data true ) " 2965 " then ( addr sfmmup ) " 2966 " loop ( addr sfmmup ) " 2967 " 2drop false ( false ) " 2968 "; " 2969 ; 2970 2971 void 2972 create_va_to_tte(void) 2973 { 2974 char *bp; 2975 extern int khmehash_num, uhmehash_num; 2976 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2977 2978 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2979 2980 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2981 2982 /* 2983 * Teach obp how to parse our sw ttes. 2984 */ 2985 (void) sprintf(bp, obp_tte_str, 2986 MMU_PAGESHIFT, 2987 TTE8K, 2988 sizeof (struct sf_hment), 2989 OFFSET(struct sf_hment, hme_tte), 2990 OFFSET(struct hme_blk, hblk_tag), 2991 OFFSET(struct hme_blk, hblk_nextpa), 2992 OFFSET(struct hme_blk, hblk_misc), 2993 OFFSET(struct hme_blk, hblk_hme), 2994 NHMENTS, 2995 HBLK_SZMASK, 2996 HBLK_RANGE_SHIFT, 2997 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2998 HMEBLK_ENDPA, 2999 sizeof (struct hmehash_bucket), 3000 HTAG_SFMMUPSZ, 3001 HTAG_BSPAGE_SHIFT, 3002 HTAG_REHASH_SHIFT, 3003 SFMMU_INVALID_SHMERID, 3004 mmu_hashcnt, 3005 (caddr_t)va_to_pa((caddr_t)uhme_hash), 3006 (caddr_t)va_to_pa((caddr_t)khme_hash), 3007 UHMEHASH_SZ, 3008 KHMEHASH_SZ, 3009 KCONTEXT, 3010 KHATID, 3011 ASI_MEM); 3012 prom_interpret(bp, 0, 0, 0, 0, 0); 3013 3014 kobj_free(bp, MMU_PAGESIZE); 3015 } 3016 3017 void 3018 install_va_to_tte(void) 3019 { 3020 /* 3021 * advise prom that it can use unix-tte 3022 */ 3023 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3024 } 3025 3026 /* 3027 * Here we add "device-type=console" for /os-io node, for currently 3028 * our kernel console output only supports displaying text and 3029 * performing cursor-positioning operations (through kernel framebuffer 3030 * driver) and it doesn't support other functionalities required for a 3031 * standard "display" device as specified in 1275 spec. The main missing 3032 * interface defined by the 1275 spec is "draw-logo". 3033 * also see the comments above prom_stdout_is_framebuffer(). 3034 */ 3035 static char *create_node = 3036 "\" /\" find-device " 3037 "new-device " 3038 "\" os-io\" device-name " 3039 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3040 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3041 " 2>r swap 2 2r> ['] $callback catch if " 3042 " 2drop 3drop 0 " 3043 " then " 3044 "; " 3045 ": read ( adr,len -- #read ) " 3046 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3047 " ( retN ... ret1 N ) " 3048 " ?dup if " 3049 " swap >r 1- 0 ?do drop loop r> " 3050 " else " 3051 " -2 " 3052 " then " 3053 "; " 3054 ": write ( adr,len -- #written ) " 3055 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3056 " ( retN ... ret1 N ) " 3057 " ?dup if " 3058 " swap >r 1- 0 ?do drop loop r> " 3059 " else " 3060 " 0 " 3061 " then " 3062 "; " 3063 ": poll-tty ( -- ) ; " 3064 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3065 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3066 ": cb-give/take ( $method -- ) " 3067 " 0 -rot ['] $callback catch ?dup if " 3068 " >r 2drop 2drop r> throw " 3069 " else " 3070 " 0 ?do drop loop " 3071 " then " 3072 "; " 3073 ": give ( -- ) \" exit-input\" cb-give/take ; " 3074 ": take ( -- ) \" enter-input\" cb-give/take ; " 3075 ": open ( -- ok? ) true ; " 3076 ": close ( -- ) ; " 3077 "finish-device " 3078 "device-end "; 3079 3080 /* 3081 * Create the OBP input/output node (FCode serial driver). 3082 * It is needed for both USB console keyboard and for 3083 * the kernel terminal emulator. It is too early to check for a 3084 * kernel console compatible framebuffer now, so we create this 3085 * so that we're ready if we need to enable kernel terminal emulation. 3086 * 3087 * When the USB software takes over the input device at the time 3088 * consconfig runs, OBP's stdin is redirected to this node. 3089 * Whenever the FORTH user interface is used after this switch, 3090 * the node will call back into the kernel for console input. 3091 * If a serial device such as ttya or a UART with a Type 5 keyboard 3092 * attached is used, OBP takes over the serial device when the system 3093 * goes to the debugger after the system is booted. This sharing 3094 * of the relatively simple serial device is difficult but possible. 3095 * Sharing the USB host controller is impossible due its complexity. 3096 * 3097 * Similarly to USB keyboard input redirection, after consconfig_dacf 3098 * configures a kernel console framebuffer as the standard output 3099 * device, OBP's stdout is switched to to vector through the 3100 * /os-io node into the kernel terminal emulator. 3101 */ 3102 static void 3103 startup_create_io_node(void) 3104 { 3105 prom_interpret(create_node, 0, 0, 0, 0, 0); 3106 } 3107 3108 3109 static void 3110 do_prom_version_check(void) 3111 { 3112 int i; 3113 pnode_t node; 3114 char buf[64]; 3115 static char drev[] = "Down-rev firmware detected%s\n" 3116 "\tPlease upgrade to the following minimum version:\n" 3117 "\t\t%s\n"; 3118 3119 i = prom_version_check(buf, sizeof (buf), &node); 3120 3121 if (i == PROM_VER64_OK) 3122 return; 3123 3124 if (i == PROM_VER64_UPGRADE) { 3125 cmn_err(CE_WARN, drev, "", buf); 3126 3127 #ifdef DEBUG 3128 prom_enter_mon(); /* Type 'go' to continue */ 3129 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3130 return; 3131 #else 3132 halt(0); 3133 #endif 3134 } 3135 3136 /* 3137 * The other possibility is that this is a server running 3138 * good firmware, but down-rev firmware was detected on at 3139 * least one other cpu board. We just complain if we see 3140 * that. 3141 */ 3142 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3143 } 3144 3145 3146 /* 3147 * Must be defined in platform dependent code. 3148 */ 3149 extern caddr_t modtext; 3150 extern size_t modtext_sz; 3151 extern caddr_t moddata; 3152 3153 #define HEAPTEXT_ARENA(addr) \ 3154 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3155 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3156 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3157 3158 #define HEAPTEXT_OVERSIZED(addr) \ 3159 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3160 3161 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3162 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3163 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3164 3165 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3166 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3167 kmutex_t texthole_lock; 3168 3169 char kern_bootargs[OBP_MAXPATHLEN]; 3170 char kern_bootfile[OBP_MAXPATHLEN]; 3171 3172 void 3173 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3174 { 3175 uintptr_t addr, limit; 3176 3177 addr = HEAPTEXT_BASE; 3178 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3179 3180 /* 3181 * Before we initialize the text_arena, we want to punch holes in the 3182 * underlying heaptext_arena. This guarantees that for any text 3183 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3184 */ 3185 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3186 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3187 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3188 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3189 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3190 } 3191 3192 /* 3193 * Allocate one page at the oversize to break up the text region 3194 * from the oversized region. 3195 */ 3196 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3197 (void *)limit, (void *)(limit + PAGESIZE), 3198 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3199 3200 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3201 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3202 heaptext_arena, 0, VM_SLEEP); 3203 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3204 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3205 } 3206 3207 caddr_t 3208 kobj_text_alloc(vmem_t *arena, size_t size) 3209 { 3210 caddr_t rval, better; 3211 3212 /* 3213 * First, try a sleeping allocation. 3214 */ 3215 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3216 3217 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3218 return (rval); 3219 3220 /* 3221 * We didn't get the area that we wanted. We're going to try to do an 3222 * allocation with explicit constraints. 3223 */ 3224 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3225 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3226 VM_NOSLEEP | VM_BESTFIT); 3227 3228 if (better != NULL) { 3229 /* 3230 * That worked. Free our first attempt and return. 3231 */ 3232 vmem_free(arena, rval, size); 3233 return (better); 3234 } 3235 3236 /* 3237 * That didn't work; we'll have to return our first attempt. 3238 */ 3239 return (rval); 3240 } 3241 3242 caddr_t 3243 kobj_texthole_alloc(caddr_t addr, size_t size) 3244 { 3245 int arena = HEAPTEXT_ARENA(addr); 3246 char c[30]; 3247 uintptr_t base; 3248 3249 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3250 /* 3251 * If this is an oversized allocation or it is allocated in 3252 * the nucleus data page, there is no text hole available for 3253 * it; return NULL. 3254 */ 3255 return (NULL); 3256 } 3257 3258 mutex_enter(&texthole_lock); 3259 3260 if (texthole_arena[arena] == NULL) { 3261 ASSERT(texthole_source[arena] == NULL); 3262 3263 if (arena == 0) { 3264 texthole_source[0] = vmem_create("module_text_holesrc", 3265 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3266 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3267 0, VM_SLEEP); 3268 } else { 3269 base = HEAPTEXT_BASE + 3270 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3271 3272 (void) snprintf(c, sizeof (c), 3273 "heaptext_holesrc_%d", arena); 3274 3275 texthole_source[arena] = vmem_create(c, (void *)base, 3276 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3277 0, VM_SLEEP); 3278 } 3279 3280 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3281 3282 texthole_arena[arena] = vmem_create(c, NULL, 0, 3283 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3284 texthole_source[arena], 0, VM_SLEEP); 3285 } 3286 3287 mutex_exit(&texthole_lock); 3288 3289 ASSERT(texthole_arena[arena] != NULL); 3290 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3291 return (vmem_alloc(texthole_arena[arena], size, 3292 VM_BESTFIT | VM_NOSLEEP)); 3293 } 3294 3295 void 3296 kobj_texthole_free(caddr_t addr, size_t size) 3297 { 3298 int arena = HEAPTEXT_ARENA(addr); 3299 3300 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3301 ASSERT(texthole_arena[arena] != NULL); 3302 vmem_free(texthole_arena[arena], addr, size); 3303 } 3304 3305 void 3306 release_bootstrap(void) 3307 { 3308 if (&cif_init) 3309 cif_init(); 3310 } 3311