1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/machsystm.h> 28 #include <sys/archsystm.h> 29 #include <sys/vm.h> 30 #include <sys/cpu.h> 31 #include <sys/atomic.h> 32 #include <sys/reboot.h> 33 #include <sys/kdi.h> 34 #include <sys/bootconf.h> 35 #include <sys/memlist_plat.h> 36 #include <sys/memlist_impl.h> 37 #include <sys/prom_plat.h> 38 #include <sys/prom_isa.h> 39 #include <sys/autoconf.h> 40 #include <sys/intreg.h> 41 #include <sys/ivintr.h> 42 #include <sys/fpu/fpusystm.h> 43 #include <sys/iommutsb.h> 44 #include <vm/vm_dep.h> 45 #include <vm/seg_dev.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_kpm.h> 48 #include <vm/seg_map.h> 49 #include <vm/seg_kp.h> 50 #include <sys/sysconf.h> 51 #include <vm/hat_sfmmu.h> 52 #include <sys/kobj.h> 53 #include <sys/sun4asi.h> 54 #include <sys/clconf.h> 55 #include <sys/platform_module.h> 56 #include <sys/panic.h> 57 #include <sys/cpu_sgnblk_defs.h> 58 #include <sys/clock.h> 59 #include <sys/cmn_err.h> 60 #include <sys/promif.h> 61 #include <sys/prom_debug.h> 62 #include <sys/traptrace.h> 63 #include <sys/memnode.h> 64 #include <sys/mem_cage.h> 65 #include <sys/mmu.h> 66 67 extern void setup_trap_table(void); 68 extern int cpu_intrq_setup(struct cpu *); 69 extern void cpu_intrq_register(struct cpu *); 70 extern void contig_mem_init(void); 71 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 72 extern void mach_dump_buffer_init(void); 73 extern void mach_descrip_init(void); 74 extern void mach_descrip_startup_fini(void); 75 extern void mach_memscrub(void); 76 extern void mach_fpras(void); 77 extern void mach_cpu_halt_idle(void); 78 extern void mach_hw_copy_limit(void); 79 extern void load_mach_drivers(void); 80 extern void load_tod_module(void); 81 #pragma weak load_tod_module 82 83 extern int ndata_alloc_mmfsa(struct memlist *ndata); 84 #pragma weak ndata_alloc_mmfsa 85 86 extern void cif_init(void); 87 #pragma weak cif_init 88 89 extern void parse_idprom(void); 90 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91 extern void mem_config_init(void); 92 extern void memseg_remap_init(void); 93 94 extern void mach_kpm_init(void); 95 extern void pcf_init(); 96 extern int size_pse_array(pgcnt_t, int); 97 extern void pg_init(); 98 99 /* 100 * External Data: 101 */ 102 extern int vac_size; /* cache size in bytes */ 103 extern uint_t vac_mask; /* VAC alignment consistency mask */ 104 extern uint_t vac_colors; 105 106 /* 107 * Global Data Definitions: 108 */ 109 110 /* 111 * XXX - Don't port this to new architectures 112 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 113 * 'romp' has no use with a prom with an IEEE 1275 client interface. 114 * The driver doesn't use the value, but it depends on the symbol. 115 */ 116 void *romp; /* veritas driver won't load without romp 4154976 */ 117 /* 118 * Declare these as initialized data so we can patch them. 119 */ 120 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 121 pgcnt_t segkpsize = 122 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 123 uint_t segmap_percent = 6; /* Size of segmap segment */ 124 125 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 126 int vac_copyback = 1; 127 char *cache_mode = NULL; 128 int use_mix = 1; 129 int prom_debug = 0; 130 131 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 132 uint_t tba_taken_over = 0; 133 134 caddr_t s_text; /* start of kernel text segment */ 135 caddr_t e_text; /* end of kernel text segment */ 136 caddr_t s_data; /* start of kernel data segment */ 137 caddr_t e_data; /* end of kernel data segment */ 138 139 caddr_t modtext; /* beginning of module text */ 140 size_t modtext_sz; /* size of module text */ 141 caddr_t moddata; /* beginning of module data reserve */ 142 caddr_t e_moddata; /* end of module data reserve */ 143 144 /* 145 * End of first block of contiguous kernel in 32-bit virtual address space 146 */ 147 caddr_t econtig32; /* end of first blk of contiguous kernel */ 148 149 caddr_t ncbase; /* beginning of non-cached segment */ 150 caddr_t ncend; /* end of non-cached segment */ 151 152 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 153 caddr_t nalloc_base; /* beginning of nucleus allocation */ 154 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 155 caddr_t valloc_base; /* beginning of kvalloc segment */ 156 157 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 158 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 159 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 160 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 161 int kmem64_szc; /* page size code */ 162 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 163 164 uintptr_t shm_alignment; /* VAC address consistency modulus */ 165 struct memlist *phys_install; /* Total installed physical memory */ 166 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 167 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 168 struct memlist *nopp_list; /* pages with no backing page structs */ 169 struct memlist ndata; /* memlist of nucleus allocatable memory */ 170 int memexp_flag; /* memory expansion card flag */ 171 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 172 pgcnt_t obp_pages; /* Physical pages used by OBP */ 173 174 /* 175 * VM data structures 176 */ 177 long page_hashsz; /* Size of page hash table (power of two) */ 178 struct page *pp_base; /* Base of system page struct array */ 179 size_t pp_sz; /* Size in bytes of page struct array */ 180 struct page **page_hash; /* Page hash table */ 181 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 182 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 183 int pse_shift; /* log2(pse_table_size) */ 184 struct seg ktextseg; /* Segment used for kernel executable image */ 185 struct seg kvalloc; /* Segment used for "valloc" mapping */ 186 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 187 struct seg ktexthole; /* Segment used for nucleus text hole */ 188 struct seg kmapseg; /* Segment used for generic kernel mappings */ 189 struct seg kpmseg; /* Segment used for physical mapping */ 190 struct seg kdebugseg; /* Segment used for the kernel debugger */ 191 192 void *kpm_pp_base; /* Base of system kpm_page array */ 193 size_t kpm_pp_sz; /* Size of system kpm_page array */ 194 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 195 196 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 197 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 198 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 199 200 int segzio_fromheap = 0; /* zio allocations occur from heap */ 201 caddr_t segzio_base; /* Base address of segzio */ 202 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 203 204 /* 205 * A static DR page_t VA map is reserved that can map the page structures 206 * for a domain's entire RA space. The pages that backs this space are 207 * dynamically allocated and need not be physically contiguous. The DR 208 * map size is derived from KPM size. 209 */ 210 int ppvm_enable = 0; /* Static virtual map for page structs */ 211 page_t *ppvm_base; /* Base of page struct map */ 212 pgcnt_t ppvm_size = 0; /* Size of page struct map */ 213 214 /* 215 * debugger pages (if allocated) 216 */ 217 struct vnode kdebugvp; 218 219 /* 220 * VA range available to the debugger 221 */ 222 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 223 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 224 225 /* 226 * Segment for relocated kernel structures in 64-bit large RAM kernels 227 */ 228 struct seg kmem64; 229 230 struct memseg *memseg_free; 231 232 struct vnode unused_pages_vp; 233 234 /* 235 * VM data structures allocated early during boot. 236 */ 237 size_t pagehash_sz; 238 uint64_t memlist_sz; 239 240 char tbr_wr_addr_inited = 0; 241 242 caddr_t mpo_heap32_buf = NULL; 243 size_t mpo_heap32_bufsz = 0; 244 245 /* 246 * Static Routines: 247 */ 248 static int ndata_alloc_memseg(struct memlist *, size_t); 249 static void memlist_new(uint64_t, uint64_t, struct memlist **); 250 static void memlist_add(uint64_t, uint64_t, 251 struct memlist **, struct memlist **); 252 static void kphysm_init(void); 253 static void kvm_init(void); 254 static void install_kmem64_tte(void); 255 256 static void startup_init(void); 257 static void startup_memlist(void); 258 static void startup_modules(void); 259 static void startup_bop_gone(void); 260 static void startup_vm(void); 261 static void startup_end(void); 262 static void setup_cage_params(void); 263 static void startup_create_io_node(void); 264 265 static pgcnt_t npages; 266 static struct memlist *memlist; 267 void *memlist_end; 268 269 static pgcnt_t bop_alloc_pages; 270 static caddr_t hblk_base; 271 uint_t hblk_alloc_dynamic = 0; 272 uint_t hblk1_min = H1MIN; 273 274 275 /* 276 * Hooks for unsupported platforms and down-rev firmware 277 */ 278 int iam_positron(void); 279 #pragma weak iam_positron 280 static void do_prom_version_check(void); 281 282 /* 283 * After receiving a thermal interrupt, this is the number of seconds 284 * to delay before shutting off the system, assuming 285 * shutdown fails. Use /etc/system to change the delay if this isn't 286 * large enough. 287 */ 288 int thermal_powerdown_delay = 1200; 289 290 /* 291 * Used to hold off page relocations into the cage until OBP has completed 292 * its boot-time handoff of its resources to the kernel. 293 */ 294 int page_relocate_ready = 0; 295 296 /* 297 * Indicate if kmem64 allocation was done in small chunks 298 */ 299 int kmem64_smchunks = 0; 300 301 /* 302 * Enable some debugging messages concerning memory usage... 303 */ 304 #ifdef DEBUGGING_MEM 305 static int debugging_mem; 306 static void 307 printmemlist(char *title, struct memlist *list) 308 { 309 if (!debugging_mem) 310 return; 311 312 printf("%s\n", title); 313 314 while (list) { 315 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 316 (uint32_t)(list->address >> 32), (uint32_t)list->address, 317 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 318 list = list->next; 319 } 320 } 321 322 void 323 printmemseg(struct memseg *memseg) 324 { 325 if (!debugging_mem) 326 return; 327 328 printf("memseg\n"); 329 330 while (memseg) { 331 prom_printf("\tpage = 0x%p, epage = 0x%p, " 332 "pfn = 0x%x, epfn = 0x%x\n", 333 memseg->pages, memseg->epages, 334 memseg->pages_base, memseg->pages_end); 335 memseg = memseg->next; 336 } 337 } 338 339 #define debug_pause(str) halt((str)) 340 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 341 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 342 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 343 #define MPRINTF3(str, a, b, c) \ 344 if (debugging_mem) prom_printf((str), (a), (b), (c)) 345 #else /* DEBUGGING_MEM */ 346 #define MPRINTF(str) 347 #define MPRINTF1(str, a) 348 #define MPRINTF2(str, a, b) 349 #define MPRINTF3(str, a, b, c) 350 #endif /* DEBUGGING_MEM */ 351 352 353 /* 354 * 355 * Kernel's Virtual Memory Layout. 356 * /-----------------------\ 357 * 0xFFFFFFFF.FFFFFFFF -| |- 358 * | OBP's virtual page | 359 * | tables | 360 * 0xFFFFFFFC.00000000 -|-----------------------|- 361 * : : 362 * : : 363 * -|-----------------------|- 364 * | segzio | (base and size vary) 365 * 0xFFFFFE00.00000000 -|-----------------------|- 366 * | | Ultrasparc I/II support 367 * | segkpm segment | up to 2TB of physical 368 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 369 * | | 370 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 371 * : : 372 * : : 373 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 374 * | | ^ 375 * | UltraSPARC I/II call | | 376 * | bug requires an extra | | 377 * | 4 GB of space between | | 378 * | hole and used RAM | | 379 * | | | 380 * 0xFFFFF800.00000000 -|-----------------------|- | 381 * | | | 382 * | Virtual Address Hole | UltraSPARC 383 * | on UltraSPARC I/II | I/II * ONLY * 384 * | | | 385 * 0x00000800.00000000 -|-----------------------|- | 386 * | | | 387 * | UltraSPARC I/II call | | 388 * | bug requires an extra | | 389 * | 4 GB of space between | | 390 * | hole and used RAM | | 391 * | | v 392 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 393 * : : ^ 394 * : : | 395 * |-----------------------| | 396 * | | | 397 * | ecache flush area | | 398 * | (twice largest e$) | | 399 * | | | 400 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 401 * | overmapped area | alignend_end | 402 * | (kmem64_alignsize | | 403 * | boundary) | | 404 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 405 * | | | 406 * | 64-bit kernel ONLY | | 407 * | | | 408 * | kmem64 segment | | 409 * | | | 410 * | (Relocated extra HME | Approximately 411 * | block allocations, | 1 TB of virtual 412 * | memnode freelists, | address space 413 * | HME hash buckets, | | 414 * | mml_table, kpmp_table,| | 415 * | page_t array and | | 416 * | hashblock pool to | | 417 * | avoid hard-coded | | 418 * | 32-bit vaddr | | 419 * | limitations) | | 420 * | | v 421 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 422 * | | 423 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 424 * | | 425 * 0x00000300.00000000 -|-----------------------|- SYSBASE 426 * : : 427 * : : 428 * -|-----------------------|- 429 * | | 430 * | segmap segment | SEGMAPSIZE (1/8th physmem, 431 * | | 256G MAX) 432 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 433 * : : 434 * : : 435 * -|-----------------------|- 436 * | | 437 * | segkp | SEGKPSIZE (2GB) 438 * | | 439 * | | 440 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 441 * | | 442 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 443 * | | (SEGKPBASE - 0x400000) 444 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 445 * | | (MEMSCRUBBASE - NCARGS) 446 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 447 * | | (ARGSBASE - PPMAPSIZE) 448 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 449 * | | 450 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 451 * | | 452 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 453 * : : 454 * : : 455 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 456 * | | 457 * | OBP | 458 * | | 459 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 460 * | kmdb | 461 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 462 * : : 463 * : : 464 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 465 * | | 466 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 467 * | | ~64MB) 468 * 0x00000000.70002000 -|-----------------------| 469 * | panicbuf | 470 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 471 * | boot-time | 472 * | temporary space | 473 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 474 * : : 475 * : : 476 * | | 477 * |-----------------------|- econtig32 478 * | vm structures | 479 * 0x00000000.01C00000 |-----------------------|- nalloc_end 480 * | TSBs | 481 * |-----------------------|- end/nalloc_base 482 * | kernel data & bss | 483 * 0x00000000.01800000 -|-----------------------| 484 * : nucleus text hole : 485 * 0x00000000.01400000 -|-----------------------| 486 * : : 487 * |-----------------------| 488 * | module text | 489 * |-----------------------|- e_text/modtext 490 * | kernel text | 491 * |-----------------------| 492 * | trap table (48k) | 493 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 494 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 495 * |-----------------------| 496 * | | 497 * | invalid | 498 * | | 499 * 0x00000000.00000000 _|_______________________| 500 * 501 * 502 * 503 * 32-bit User Virtual Memory Layout. 504 * /-----------------------\ 505 * | | 506 * | invalid | 507 * | | 508 * 0xFFC00000 -|-----------------------|- USERLIMIT 509 * | user stack | 510 * : : 511 * : : 512 * : : 513 * | user data | 514 * -|-----------------------|- 515 * | user text | 516 * 0x00002000 -|-----------------------|- 517 * | invalid | 518 * 0x00000000 _|_______________________| 519 * 520 * 521 * 522 * 64-bit User Virtual Memory Layout. 523 * /-----------------------\ 524 * | | 525 * | invalid | 526 * | | 527 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 528 * | user stack | 529 * : : 530 * : : 531 * : : 532 * | user data | 533 * -|-----------------------|- 534 * | user text | 535 * 0x00000000.01000000 -|-----------------------|- 536 * | invalid | 537 * 0x00000000.00000000 _|_______________________| 538 */ 539 540 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 541 extern uint64_t ecache_flush_address(void); 542 543 #pragma weak load_platform_modules 544 #pragma weak plat_startup_memlist 545 #pragma weak ecache_init_scrub_flush_area 546 #pragma weak ecache_flush_address 547 548 549 /* 550 * By default the DR Cage is enabled for maximum OS 551 * MPSS performance. Users needing to disable the cage mechanism 552 * can set this variable to zero via /etc/system. 553 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 554 * will result in loss of DR functionality. 555 * Platforms wishing to disable kernel Cage by default 556 * should do so in their set_platform_defaults() routine. 557 */ 558 int kernel_cage_enable = 1; 559 560 static void 561 setup_cage_params(void) 562 { 563 void (*func)(void); 564 565 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 566 if (func != NULL) { 567 (*func)(); 568 return; 569 } 570 571 if (kernel_cage_enable == 0) { 572 return; 573 } 574 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 575 576 if (kcage_on) { 577 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 578 } else { 579 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 580 } 581 582 } 583 584 /* 585 * Machine-dependent startup code 586 */ 587 void 588 startup(void) 589 { 590 startup_init(); 591 if (&startup_platform) 592 startup_platform(); 593 startup_memlist(); 594 startup_modules(); 595 setup_cage_params(); 596 startup_bop_gone(); 597 startup_vm(); 598 startup_end(); 599 } 600 601 struct regs sync_reg_buf; 602 uint64_t sync_tt; 603 604 void 605 sync_handler(void) 606 { 607 struct panic_trap_info ti; 608 int i; 609 610 /* 611 * Prevent trying to talk to the other CPUs since they are 612 * sitting in the prom and won't reply. 613 */ 614 for (i = 0; i < NCPU; i++) { 615 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 616 cpu[i]->cpu_flags &= ~CPU_READY; 617 cpu[i]->cpu_flags |= CPU_QUIESCED; 618 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 619 } 620 } 621 622 /* 623 * We've managed to get here without going through the 624 * normal panic code path. Try and save some useful 625 * information. 626 */ 627 if (!panicstr && (curthread->t_panic_trap == NULL)) { 628 ti.trap_type = sync_tt; 629 ti.trap_regs = &sync_reg_buf; 630 ti.trap_addr = NULL; 631 ti.trap_mmu_fsr = 0x0; 632 633 curthread->t_panic_trap = &ti; 634 } 635 636 /* 637 * If we're re-entering the panic path, update the signature 638 * block so that the SC knows we're in the second part of panic. 639 */ 640 if (panicstr) 641 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 642 643 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 644 panic("sync initiated"); 645 } 646 647 648 static void 649 startup_init(void) 650 { 651 /* 652 * We want to save the registers while we're still in OBP 653 * so that we know they haven't been fiddled with since. 654 * (In principle, OBP can't change them just because it 655 * makes a callback, but we'd rather not depend on that 656 * behavior.) 657 */ 658 char sync_str[] = 659 "warning @ warning off : sync " 660 "%%tl-c %%tstate h# %p x! " 661 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 662 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 663 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 664 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 665 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 666 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 667 "%%y h# %p l! %%tl-c %%tt h# %p x! " 668 "sync ; warning !"; 669 670 /* 671 * 20 == num of %p substrings 672 * 16 == max num of chars %p will expand to. 673 */ 674 char bp[sizeof (sync_str) + 16 * 20]; 675 676 /* 677 * Initialize ptl1 stack for the 1st CPU. 678 */ 679 ptl1_init_cpu(&cpu0); 680 681 /* 682 * Initialize the address map for cache consistent mappings 683 * to random pages; must be done after vac_size is set. 684 */ 685 ppmapinit(); 686 687 /* 688 * Initialize the PROM callback handler. 689 */ 690 init_vx_handler(); 691 692 /* 693 * have prom call sync_callback() to handle the sync and 694 * save some useful information which will be stored in the 695 * core file later. 696 */ 697 (void) sprintf((char *)bp, sync_str, 698 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 699 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 700 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 701 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 702 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 703 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 704 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 705 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 706 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 707 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 708 prom_interpret(bp, 0, 0, 0, 0, 0); 709 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 710 } 711 712 713 size_t 714 calc_pp_sz(pgcnt_t npages) 715 { 716 717 return (npages * sizeof (struct page)); 718 } 719 720 size_t 721 calc_kpmpp_sz(pgcnt_t npages) 722 { 723 724 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 725 kpm_pgsz = 1ull << kpm_pgshft; 726 kpm_pgoff = kpm_pgsz - 1; 727 kpmp2pshft = kpm_pgshft - PAGESHIFT; 728 kpmpnpgs = 1 << kpmp2pshft; 729 730 if (kpm_smallpages == 0) { 731 /* 732 * Avoid fragmentation problems in kphysm_init() 733 * by allocating for all of physical memory 734 */ 735 kpm_npages = ptokpmpr(physinstalled); 736 return (kpm_npages * sizeof (kpm_page_t)); 737 } else { 738 kpm_npages = npages; 739 return (kpm_npages * sizeof (kpm_spage_t)); 740 } 741 } 742 743 size_t 744 calc_pagehash_sz(pgcnt_t npages) 745 { 746 747 /* 748 * The page structure hash table size is a power of 2 749 * such that the average hash chain length is PAGE_HASHAVELEN. 750 */ 751 page_hashsz = npages / PAGE_HASHAVELEN; 752 page_hashsz = 1 << highbit(page_hashsz); 753 return (page_hashsz * sizeof (struct page *)); 754 } 755 756 int testkmem64_smchunks = 0; 757 758 int 759 alloc_kmem64(caddr_t base, caddr_t end) 760 { 761 int i; 762 caddr_t aligned_end = NULL; 763 764 if (testkmem64_smchunks) 765 return (1); 766 767 /* 768 * Make one large memory alloc after figuring out the 64-bit size. This 769 * will enable use of the largest page size appropriate for the system 770 * architecture. 771 */ 772 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 773 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 774 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 775 size_t alloc_size, alignsize; 776 #if !defined(C_OBP) 777 unsigned long long pa; 778 #endif /* !C_OBP */ 779 780 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 781 continue; 782 alignsize = TTEBYTES(i); 783 kmem64_szc = i; 784 785 /* limit page size for small memory */ 786 if (mmu_btop(alignsize) > (npages >> 2)) 787 continue; 788 789 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 790 alloc_size = aligned_end - base; 791 #if !defined(C_OBP) 792 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 793 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 794 kmem64_pabase = pa; 795 kmem64_aligned_end = aligned_end; 796 install_kmem64_tte(); 797 break; 798 } else { 799 prom_free_phys(alloc_size, pa); 800 } 801 } 802 #else /* !C_OBP */ 803 if (prom_alloc(base, alloc_size, alignsize) == base) { 804 kmem64_pabase = va_to_pa(kmem64_base); 805 kmem64_aligned_end = aligned_end; 806 break; 807 } 808 #endif /* !C_OBP */ 809 if (i == TTE8K) { 810 #ifdef sun4v 811 /* return failure to try small allocations */ 812 return (1); 813 #else 814 prom_panic("kmem64 allocation failure"); 815 #endif 816 } 817 } 818 ASSERT(aligned_end != NULL); 819 return (0); 820 } 821 822 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 823 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 824 825 #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 826 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 827 (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 828 829 #if !defined(C_OBP) 830 /* 831 * Install a temporary tte handler in OBP for kmem64 area. 832 * 833 * We map kmem64 area with large pages before the trap table is taken 834 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 835 * the same area. Duplicate tlb entries with different page sizes 836 * cause unpredicatble behavior. To avoid this, we don't create 837 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 838 * OBP). Instead, we manage translations with a temporary va>tte-data 839 * handler (kmem64-tte). This handler is replaced by unix-tte when 840 * the trap table is taken over. 841 * 842 * The temporary handler knows the physical address of the kmem64 843 * area. It uses the prom's pgmap@ Forth word for other addresses. 844 * 845 * We have to use BOP_ALLOC() method for C-OBP platforms because 846 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 847 * sun4u platforms. On sun4u we flush tlb after trap table is taken 848 * over if we use large pages for kernel heap and kmem64. Since sun4u 849 * prom (unlike sun4v) calls va>tte-data first for client address 850 * translation prom's ttes for kmem64 can't get into TLB even if we 851 * later switch to prom's trap table again. C-OBP uses 4M pages for 852 * client mappings when possible so on all platforms we get the 853 * benefit from large mappings for kmem64 area immediately during 854 * boot. 855 * 856 * pseudo code: 857 * if (context != 0) { 858 * return false 859 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 860 * tte = tte_template + 861 * (((miss_va & pagemask) - kmem64_base)); 862 * return tte, true 863 * } else { 864 * return pgmap@ result 865 * } 866 */ 867 char kmem64_obp_str[] = 868 "h# %lx constant kmem64-base " 869 "h# %lx constant kmem64-end " 870 "h# %lx constant kmem64-pagemask " 871 "h# %lx constant kmem64-template " 872 873 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 874 " if ( addr ) " 875 " drop false exit then ( false ) " 876 " dup kmem64-base kmem64-end within if ( addr ) " 877 " kmem64-pagemask and ( addr' ) " 878 " kmem64-base - ( addr' ) " 879 " kmem64-template + ( tte ) " 880 " true ( tte true ) " 881 " else ( addr ) " 882 " pgmap@ ( tte ) " 883 " dup 0< if true else drop false then ( tte true | false ) " 884 " then ( tte true | false ) " 885 "; " 886 887 "' kmem64-tte is va>tte-data " 888 ; 889 890 static void 891 install_kmem64_tte() 892 { 893 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 894 tte_t tte; 895 896 PRM_DEBUG(kmem64_pabase); 897 PRM_DEBUG(kmem64_szc); 898 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 899 PROC_DATA | HAT_NOSYNC, kmem64_szc); 900 PRM_DEBUG(tte.ll); 901 (void) sprintf(b, kmem64_obp_str, 902 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 903 ASSERT(strlen(b) < sizeof (b)); 904 prom_interpret(b, 0, 0, 0, 0, 0); 905 } 906 #endif /* !C_OBP */ 907 908 /* 909 * As OBP takes up some RAM when the system boots, pages will already be "lost" 910 * to the system and reflected in npages by the time we see it. 911 * 912 * We only want to allocate kernel structures in the 64-bit virtual address 913 * space on systems with enough RAM to make the overhead of keeping track of 914 * an extra kernel memory segment worthwhile. 915 * 916 * Since OBP has already performed its memory allocations by this point, if we 917 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 918 * memory in the 64-bit virtual address space; otherwise keep allocations 919 * contiguous with we've mapped so far in the 32-bit virtual address space. 920 */ 921 #define MINMOVE_RAM_MB ((size_t)1900) 922 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 923 #define BYTES_TO_MB(b) ((b) / 1048576ul) 924 925 pgcnt_t tune_npages = (pgcnt_t) 926 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 927 928 #pragma weak page_set_colorequiv_arr_cpu 929 extern void page_set_colorequiv_arr_cpu(void); 930 extern void page_set_colorequiv_arr(void); 931 932 static pgcnt_t ramdisk_npages; 933 static struct memlist *old_phys_avail; 934 935 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 936 937 static void 938 startup_memlist(void) 939 { 940 size_t hmehash_sz, pagelist_sz, tt_sz; 941 size_t psetable_sz; 942 caddr_t alloc_base; 943 caddr_t memspace; 944 struct memlist *cur; 945 size_t syslimit = (size_t)SYSLIMIT; 946 size_t sysbase = (size_t)SYSBASE; 947 948 /* 949 * Initialize enough of the system to allow kmem_alloc to work by 950 * calling boot to allocate its memory until the time that 951 * kvm_init is completed. The page structs are allocated after 952 * rounding up end to the nearest page boundary; the memsegs are 953 * initialized and the space they use comes from the kernel heap. 954 * With appropriate initialization, they can be reallocated later 955 * to a size appropriate for the machine's configuration. 956 * 957 * At this point, memory is allocated for things that will never 958 * need to be freed, this used to be "valloced". This allows a 959 * savings as the pages don't need page structures to describe 960 * them because them will not be managed by the vm system. 961 */ 962 963 /* 964 * We're loaded by boot with the following configuration (as 965 * specified in the sun4u/conf/Mapfile): 966 * 967 * text: 4 MB chunk aligned on a 4MB boundary 968 * data & bss: 4 MB chunk aligned on a 4MB boundary 969 * 970 * These two chunks will eventually be mapped by 2 locked 4MB 971 * ttes and will represent the nucleus of the kernel. This gives 972 * us some free space that is already allocated, some or all of 973 * which is made available to kernel module text. 974 * 975 * The free space in the data-bss chunk is used for nucleus 976 * allocatable data structures and we reserve it using the 977 * nalloc_base and nalloc_end variables. This space is currently 978 * being used for hat data structures required for tlb miss 979 * handling operations. We align nalloc_base to a l2 cache 980 * linesize because this is the line size the hardware uses to 981 * maintain cache coherency. 982 * 512K is carved out for module data. 983 */ 984 985 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 986 e_moddata = moddata + MODDATA; 987 nalloc_base = e_moddata; 988 989 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 990 valloc_base = nalloc_base; 991 992 /* 993 * Calculate the start of the data segment. 994 */ 995 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 996 prom_panic("nucleus data overflow"); 997 998 PRM_DEBUG(moddata); 999 PRM_DEBUG(nalloc_base); 1000 PRM_DEBUG(nalloc_end); 1001 1002 /* 1003 * Remember any slop after e_text so we can give it to the modules. 1004 */ 1005 PRM_DEBUG(e_text); 1006 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1007 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1008 prom_panic("nucleus text overflow"); 1009 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1010 modtext; 1011 PRM_DEBUG(modtext); 1012 PRM_DEBUG(modtext_sz); 1013 1014 init_boot_memlists(); 1015 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1016 &boot_physavail, &boot_physavail_len, 1017 &boot_virtavail, &boot_virtavail_len); 1018 1019 /* 1020 * Remember what the physically available highest page is 1021 * so that dumpsys works properly, and find out how much 1022 * memory is installed. 1023 */ 1024 installed_top_size_memlist_array(boot_physinstalled, 1025 boot_physinstalled_len, &physmax, &physinstalled); 1026 PRM_DEBUG(physinstalled); 1027 PRM_DEBUG(physmax); 1028 1029 /* Fill out memory nodes config structure */ 1030 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1031 1032 /* 1033 * npages is the maximum of available physical memory possible. 1034 * (ie. it will never be more than this) 1035 * 1036 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1037 * using phys_avail will underestimate what will end up being freed. 1038 * A better initial guess is just total memory minus the kernel text 1039 */ 1040 npages = physinstalled - btop(MMU_PAGESIZE4M); 1041 1042 /* 1043 * First allocate things that can go in the nucleus data page 1044 * (fault status, TSBs, dmv, CPUs) 1045 */ 1046 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1047 1048 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1049 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1050 1051 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1052 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1053 1054 if (ndata_alloc_dmv(&ndata) != 0) 1055 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1056 1057 if (ndata_alloc_page_mutexs(&ndata) != 0) 1058 cmn_err(CE_PANIC, 1059 "no more nucleus memory after page free lists alloc"); 1060 1061 if (ndata_alloc_hat(&ndata, npages) != 0) 1062 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1063 1064 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1065 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1066 1067 /* 1068 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1069 * 1070 * There are comments all over the SFMMU code warning of dire 1071 * consequences if the TSBs are moved out of 32-bit space. This 1072 * is largely because the asm code uses "sethi %hi(addr)"-type 1073 * instructions which will not provide the expected result if the 1074 * address is a 64-bit one. 1075 * 1076 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1077 */ 1078 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1079 PRM_DEBUG(alloc_base); 1080 1081 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1082 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1083 PRM_DEBUG(alloc_base); 1084 1085 /* 1086 * Allocate IOMMU TSB array. We do this here so that the physical 1087 * memory gets deducted from the PROM's physical memory list. 1088 */ 1089 alloc_base = iommu_tsb_init(alloc_base); 1090 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1091 PRM_DEBUG(alloc_base); 1092 1093 /* 1094 * Allow for an early allocation of physically contiguous memory. 1095 */ 1096 alloc_base = contig_mem_prealloc(alloc_base, npages); 1097 1098 /* 1099 * Platforms like Starcat and OPL need special structures assigned in 1100 * 32-bit virtual address space because their probing routines execute 1101 * FCode, and FCode can't handle 64-bit virtual addresses... 1102 */ 1103 if (&plat_startup_memlist) { 1104 alloc_base = plat_startup_memlist(alloc_base); 1105 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1106 ecache_alignsize); 1107 PRM_DEBUG(alloc_base); 1108 } 1109 1110 /* 1111 * Save off where the contiguous allocations to date have ended 1112 * in econtig32. 1113 */ 1114 econtig32 = alloc_base; 1115 PRM_DEBUG(econtig32); 1116 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1117 cmn_err(CE_PANIC, "econtig32 too big"); 1118 1119 pp_sz = calc_pp_sz(npages); 1120 PRM_DEBUG(pp_sz); 1121 if (kpm_enable) { 1122 kpm_pp_sz = calc_kpmpp_sz(npages); 1123 PRM_DEBUG(kpm_pp_sz); 1124 } 1125 1126 hmehash_sz = calc_hmehash_sz(npages); 1127 PRM_DEBUG(hmehash_sz); 1128 1129 pagehash_sz = calc_pagehash_sz(npages); 1130 PRM_DEBUG(pagehash_sz); 1131 1132 pagelist_sz = calc_free_pagelist_sz(); 1133 PRM_DEBUG(pagelist_sz); 1134 1135 #ifdef TRAPTRACE 1136 tt_sz = calc_traptrace_sz(); 1137 PRM_DEBUG(tt_sz); 1138 #else 1139 tt_sz = 0; 1140 #endif /* TRAPTRACE */ 1141 1142 /* 1143 * Place the array that protects pp->p_selock in the kmem64 wad. 1144 */ 1145 pse_shift = size_pse_array(npages, max_ncpus); 1146 PRM_DEBUG(pse_shift); 1147 pse_table_size = 1 << pse_shift; 1148 PRM_DEBUG(pse_table_size); 1149 psetable_sz = roundup( 1150 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1151 PRM_DEBUG(psetable_sz); 1152 1153 /* 1154 * Now allocate the whole wad 1155 */ 1156 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1157 pagelist_sz + tt_sz + psetable_sz; 1158 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1159 kmem64_base = (caddr_t)syslimit; 1160 kmem64_end = kmem64_base + kmem64_sz; 1161 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1162 /* 1163 * Attempt for kmem64 to allocate one big 1164 * contiguous chunk of memory failed. 1165 * We get here because we are sun4v. 1166 * We will proceed by breaking up 1167 * the allocation into two attempts. 1168 * First, we allocate kpm_pp_sz, hmehash_sz, 1169 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1170 * one contiguous chunk. This is a much smaller 1171 * chunk and we should get it, if not we panic. 1172 * Note that hmehash and tt need to be physically 1173 * (in the real address sense) contiguous. 1174 * Next, we use bop_alloc_chunk() to 1175 * to allocate the page_t structures. 1176 * This will allow the page_t to be allocated 1177 * in multiple smaller chunks. 1178 * In doing so, the assumption that page_t is 1179 * physically contiguous no longer hold, this is ok 1180 * for sun4v but not for sun4u. 1181 */ 1182 size_t tmp_size; 1183 caddr_t tmp_base; 1184 1185 pp_sz = roundup(pp_sz, PAGESIZE); 1186 1187 /* 1188 * Allocate kpm_pp_sz, hmehash_sz, 1189 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1190 */ 1191 tmp_base = kmem64_base + pp_sz; 1192 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1193 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1194 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1195 prom_panic("kmem64 prom_alloc contig failed"); 1196 PRM_DEBUG(tmp_base); 1197 PRM_DEBUG(tmp_size); 1198 1199 /* 1200 * Allocate the page_ts 1201 */ 1202 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1203 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1204 PRM_DEBUG(kmem64_base); 1205 PRM_DEBUG(pp_sz); 1206 1207 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1208 ASSERT(kmem64_aligned_end >= kmem64_end); 1209 1210 kmem64_smchunks = 1; 1211 } else { 1212 1213 /* 1214 * We need to adjust pp_sz for the normal 1215 * case where kmem64 can allocate one large chunk 1216 */ 1217 if (kpm_smallpages == 0) { 1218 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1219 } else { 1220 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1221 sizeof (kpm_spage_t)); 1222 } 1223 pp_sz = npages * sizeof (struct page); 1224 } 1225 1226 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1227 cmn_err(CE_PANIC, "not enough kmem64 space"); 1228 PRM_DEBUG(kmem64_base); 1229 PRM_DEBUG(kmem64_end); 1230 PRM_DEBUG(kmem64_aligned_end); 1231 1232 /* 1233 * ... and divy it up 1234 */ 1235 alloc_base = kmem64_base; 1236 1237 pp_base = (page_t *)alloc_base; 1238 alloc_base += pp_sz; 1239 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1240 PRM_DEBUG(pp_base); 1241 PRM_DEBUG(npages); 1242 1243 if (kpm_enable) { 1244 kpm_pp_base = alloc_base; 1245 if (kpm_smallpages == 0) { 1246 /* kpm_npages based on physinstalled, don't reset */ 1247 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1248 } else { 1249 kpm_npages = ptokpmpr(npages); 1250 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1251 } 1252 alloc_base += kpm_pp_sz; 1253 alloc_base = 1254 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1255 PRM_DEBUG(kpm_pp_base); 1256 } 1257 1258 alloc_base = alloc_hmehash(alloc_base); 1259 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1260 PRM_DEBUG(alloc_base); 1261 1262 page_hash = (page_t **)alloc_base; 1263 alloc_base += pagehash_sz; 1264 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1265 PRM_DEBUG(page_hash); 1266 1267 alloc_base = alloc_page_freelists(alloc_base); 1268 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1269 PRM_DEBUG(alloc_base); 1270 1271 #ifdef TRAPTRACE 1272 ttrace_buf = alloc_base; 1273 alloc_base += tt_sz; 1274 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1275 PRM_DEBUG(alloc_base); 1276 #endif /* TRAPTRACE */ 1277 1278 pse_mutex = (pad_mutex_t *)alloc_base; 1279 alloc_base += psetable_sz; 1280 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1281 PRM_DEBUG(alloc_base); 1282 1283 /* 1284 * Note that if we use small chunk allocations for 1285 * kmem64, we need to ensure kmem64_end is the same as 1286 * kmem64_aligned_end to prevent subsequent logic from 1287 * trying to reuse the overmapping. 1288 * Otherwise we adjust kmem64_end to what we really allocated. 1289 */ 1290 if (kmem64_smchunks) { 1291 kmem64_end = kmem64_aligned_end; 1292 } else { 1293 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1294 } 1295 kmem64_sz = kmem64_end - kmem64_base; 1296 1297 if (&ecache_init_scrub_flush_area) { 1298 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1299 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1300 } 1301 1302 /* 1303 * If physmem is patched to be non-zero, use it instead of 1304 * the monitor value unless physmem is larger than the total 1305 * amount of memory on hand. 1306 */ 1307 if (physmem == 0 || physmem > npages) 1308 physmem = npages; 1309 1310 /* 1311 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1312 * is mounted as root. This memory is held down by OBP and unlike 1313 * the stub boot_archive is never released. 1314 * 1315 * In order to get things sized correctly on lower memory 1316 * machines (where the memory used by the ramdisk represents 1317 * a significant portion of memory), physmem is adjusted. 1318 * 1319 * This is done by subtracting the ramdisk_size which is set 1320 * to the size of the ramdisk (in Kb) in /etc/system at the 1321 * time the miniroot archive is constructed. 1322 */ 1323 if (root_is_ramdisk == B_TRUE) { 1324 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1325 physmem -= ramdisk_npages; 1326 } 1327 1328 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1329 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1330 1331 /* 1332 * Allocate space for the interrupt vector table. 1333 */ 1334 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1335 if (memspace != (caddr_t)intr_vec_table) 1336 prom_panic("interrupt vector table allocation failure"); 1337 1338 /* 1339 * Between now and when we finish copying in the memory lists, 1340 * allocations happen so the space gets fragmented and the 1341 * lists longer. Leave enough space for lists twice as 1342 * long as we have now; then roundup to a pagesize. 1343 */ 1344 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1345 prom_phys_avail_len() + prom_virt_avail_len()); 1346 memlist_sz *= 2; 1347 memlist_sz = roundup(memlist_sz, PAGESIZE); 1348 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1349 if (memspace == NULL) 1350 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1351 1352 memlist = (struct memlist *)memspace; 1353 memlist_end = (char *)memspace + memlist_sz; 1354 PRM_DEBUG(memlist); 1355 PRM_DEBUG(memlist_end); 1356 1357 PRM_DEBUG(sysbase); 1358 PRM_DEBUG(syslimit); 1359 kernelheap_init((void *)sysbase, (void *)syslimit, 1360 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1361 1362 /* 1363 * Take the most current snapshot we can by calling mem-update. 1364 */ 1365 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1366 &boot_physavail, &boot_physavail_len, 1367 &boot_virtavail, &boot_virtavail_len); 1368 1369 /* 1370 * Remove the space used by prom_alloc from the kernel heap 1371 * plus the area actually used by the OBP (if any) 1372 * ignoring virtual addresses in virt_avail, above syslimit. 1373 */ 1374 virt_avail = memlist; 1375 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1376 1377 for (cur = virt_avail; cur->next; cur = cur->next) { 1378 uint64_t range_base, range_size; 1379 1380 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1381 continue; 1382 if (range_base >= (uint64_t)syslimit) 1383 break; 1384 /* 1385 * Limit the range to end at syslimit. 1386 */ 1387 range_size = MIN(cur->next->address, 1388 (uint64_t)syslimit) - range_base; 1389 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1390 0, 0, (void *)range_base, (void *)(range_base + range_size), 1391 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1392 } 1393 1394 phys_avail = memlist; 1395 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1396 1397 /* 1398 * Add any extra memory at the end of the ndata region if there's at 1399 * least a page to add. There might be a few more pages available in 1400 * the middle of the ndata region, but for now they are ignored. 1401 */ 1402 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1403 if (nalloc_base == NULL) 1404 nalloc_base = nalloc_end; 1405 ndata_remain_sz = nalloc_end - nalloc_base; 1406 1407 /* 1408 * Copy physinstalled list into kernel space. 1409 */ 1410 phys_install = memlist; 1411 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1412 1413 /* 1414 * Create list of physical addrs we don't need pp's for: 1415 * kernel text 4M page 1416 * kernel data 4M page - ndata_remain_sz 1417 * kmem64 pages 1418 * 1419 * NB if adding any pages here, make sure no kpm page 1420 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1421 */ 1422 nopp_list = memlist; 1423 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1424 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1425 &memlist, &nopp_list); 1426 1427 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1428 if (!kmem64_smchunks) 1429 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1430 1431 if ((caddr_t)memlist > (memspace + memlist_sz)) 1432 prom_panic("memlist overflow"); 1433 1434 /* 1435 * Size the pcf array based on the number of cpus in the box at 1436 * boot time. 1437 */ 1438 pcf_init(); 1439 1440 /* 1441 * Initialize the page structures from the memory lists. 1442 */ 1443 kphysm_init(); 1444 1445 availrmem_initial = availrmem = freemem; 1446 PRM_DEBUG(availrmem); 1447 1448 /* 1449 * Some of the locks depend on page_hashsz being set! 1450 * kmem_init() depends on this; so, keep it here. 1451 */ 1452 page_lock_init(); 1453 1454 /* 1455 * Initialize kernel memory allocator. 1456 */ 1457 kmem_init(); 1458 1459 /* 1460 * Factor in colorequiv to check additional 'equivalent' bins 1461 */ 1462 if (&page_set_colorequiv_arr_cpu != NULL) 1463 page_set_colorequiv_arr_cpu(); 1464 else 1465 page_set_colorequiv_arr(); 1466 1467 /* 1468 * Initialize bp_mapin(). 1469 */ 1470 bp_init(shm_alignment, HAT_STRICTORDER); 1471 1472 /* 1473 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1474 * vector data structures and MPO mblock structs from the 32-bit heap. 1475 */ 1476 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1477 panicbuf, panicbuf + PANICBUFSIZE, 1478 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1479 1480 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1481 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1482 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1483 1484 if (mpo_heap32_bufsz > (size_t)0) { 1485 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1486 PAGESIZE, 0, 0, mpo_heap32_buf, 1487 mpo_heap32_buf + mpo_heap32_bufsz, 1488 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1489 } 1490 mem_config_init(); 1491 } 1492 1493 static void 1494 startup_modules(void) 1495 { 1496 int nhblk1, nhblk8; 1497 size_t nhblksz; 1498 pgcnt_t pages_per_hblk; 1499 size_t hme8blk_sz, hme1blk_sz; 1500 1501 /* 1502 * Let the platforms have a chance to change default 1503 * values before reading system file. 1504 */ 1505 if (&set_platform_defaults) 1506 set_platform_defaults(); 1507 1508 /* 1509 * Calculate default settings of system parameters based upon 1510 * maxusers, yet allow to be overridden via the /etc/system file. 1511 */ 1512 param_calc(0); 1513 1514 mod_setup(); 1515 1516 /* 1517 * If this is a positron, complain and halt. 1518 */ 1519 if (&iam_positron && iam_positron()) { 1520 cmn_err(CE_WARN, "This hardware platform is not supported" 1521 " by this release of Solaris.\n"); 1522 #ifdef DEBUG 1523 prom_enter_mon(); /* Type 'go' to resume */ 1524 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1525 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1526 1527 #else /* DEBUG */ 1528 halt(0); 1529 #endif /* DEBUG */ 1530 } 1531 1532 /* 1533 * If we are running firmware that isn't 64-bit ready 1534 * then complain and halt. 1535 */ 1536 do_prom_version_check(); 1537 1538 /* 1539 * Initialize system parameters 1540 */ 1541 param_init(); 1542 1543 /* 1544 * maxmem is the amount of physical memory we're playing with. 1545 */ 1546 maxmem = physmem; 1547 1548 /* Set segkp limits. */ 1549 ncbase = kdi_segdebugbase; 1550 ncend = kdi_segdebugbase; 1551 1552 /* 1553 * Initialize the hat layer. 1554 */ 1555 hat_init(); 1556 1557 /* 1558 * Initialize segment management stuff. 1559 */ 1560 seg_init(); 1561 1562 /* 1563 * Create the va>tte handler, so the prom can understand 1564 * kernel translations. The handler is installed later, just 1565 * as we are about to take over the trap table from the prom. 1566 */ 1567 create_va_to_tte(); 1568 1569 /* 1570 * Load the forthdebugger (optional) 1571 */ 1572 forthdebug_init(); 1573 1574 /* 1575 * Create OBP node for console input callbacks 1576 * if it is needed. 1577 */ 1578 startup_create_io_node(); 1579 1580 if (modloadonly("fs", "specfs") == -1) 1581 halt("Can't load specfs"); 1582 1583 if (modloadonly("fs", "devfs") == -1) 1584 halt("Can't load devfs"); 1585 1586 if (modloadonly("misc", "swapgeneric") == -1) 1587 halt("Can't load swapgeneric"); 1588 1589 (void) modloadonly("sys", "lbl_edition"); 1590 1591 dispinit(); 1592 1593 /* 1594 * Infer meanings to the members of the idprom buffer. 1595 */ 1596 parse_idprom(); 1597 1598 /* Read cluster configuration data. */ 1599 clconf_init(); 1600 1601 setup_ddi(); 1602 1603 /* 1604 * Lets take this opportunity to load the root device. 1605 */ 1606 if (loadrootmodules() != 0) 1607 debug_enter("Can't load the root filesystem"); 1608 1609 /* 1610 * Load tod driver module for the tod part found on this system. 1611 * Recompute the cpu frequency/delays based on tod as tod part 1612 * tends to keep time more accurately. 1613 */ 1614 if (&load_tod_module) 1615 load_tod_module(); 1616 1617 /* 1618 * Allow platforms to load modules which might 1619 * be needed after bootops are gone. 1620 */ 1621 if (&load_platform_modules) 1622 load_platform_modules(); 1623 1624 setcpudelay(); 1625 1626 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1627 &boot_physavail, &boot_physavail_len, 1628 &boot_virtavail, &boot_virtavail_len); 1629 1630 /* 1631 * Calculation and allocation of hmeblks needed to remap 1632 * the memory allocated by PROM till now. 1633 * Overestimate the number of hblk1 elements by assuming 1634 * worst case of TTE64K mappings. 1635 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1636 */ 1637 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1638 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1639 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1640 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1641 1642 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1643 1644 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1645 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1646 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1647 1648 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1649 1650 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1651 nhblk8 = 0; 1652 while (bop_alloc_pages > 1) { 1653 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1654 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1655 bop_alloc_pages *= hme8blk_sz; 1656 bop_alloc_pages = btopr(bop_alloc_pages); 1657 } 1658 nhblk8 += 2; 1659 1660 /* 1661 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1662 * boundary, the number of hblk8's needed to map the entries in the 1663 * boot_virtavail list needs to be adjusted to take this into 1664 * consideration. Thus, we need to add additional hblk8's since it 1665 * is possible that an hblk8 will not have all 8 slots used due to 1666 * alignment constraints. Since there were boot_virtavail_len entries 1667 * in that list, we need to add that many hblk8's to the number 1668 * already calculated to make sure we don't underestimate. 1669 */ 1670 nhblk8 += boot_virtavail_len; 1671 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1672 1673 /* Allocate in pagesize chunks */ 1674 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1675 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1676 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1677 } 1678 1679 static void 1680 startup_bop_gone(void) 1681 { 1682 1683 /* 1684 * Destroy the MD initialized at startup 1685 * The startup initializes the MD framework 1686 * using prom and BOP alloc free it now. 1687 */ 1688 mach_descrip_startup_fini(); 1689 1690 /* 1691 * We're done with prom allocations. 1692 */ 1693 bop_fini(); 1694 1695 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1696 &boot_physavail, &boot_physavail_len, 1697 &boot_virtavail, &boot_virtavail_len); 1698 1699 /* 1700 * setup physically contiguous area twice as large as the ecache. 1701 * this is used while doing displacement flush of ecaches 1702 */ 1703 if (&ecache_flush_address) { 1704 ecache_flushaddr = ecache_flush_address(); 1705 if (ecache_flushaddr == (uint64_t)-1) { 1706 cmn_err(CE_PANIC, 1707 "startup: no memory to set ecache_flushaddr"); 1708 } 1709 } 1710 1711 /* 1712 * Virtual available next. 1713 */ 1714 ASSERT(virt_avail != NULL); 1715 memlist_free_list(virt_avail); 1716 virt_avail = memlist; 1717 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1718 1719 } 1720 1721 1722 /* 1723 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1724 * allocations have been performed. We can't call it in startup_bop_gone 1725 * since later operations can cause obp to allocate more memory. 1726 */ 1727 void 1728 startup_fixup_physavail(void) 1729 { 1730 struct memlist *cur; 1731 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1732 1733 PRM_DEBUG(kmem64_overmap_size); 1734 1735 /* 1736 * take the most current snapshot we can by calling mem-update 1737 */ 1738 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1739 &boot_physavail, &boot_physavail_len, 1740 &boot_virtavail, &boot_virtavail_len); 1741 1742 /* 1743 * Copy phys_avail list, again. 1744 * Both the kernel/boot and the prom have been allocating 1745 * from the original list we copied earlier. 1746 */ 1747 cur = memlist; 1748 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1749 1750 /* 1751 * Add any unused kmem64 memory from overmapped page 1752 * (Note: va_to_pa does not work for kmem64_end) 1753 */ 1754 if (kmem64_overmap_size) { 1755 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1756 kmem64_overmap_size, &memlist, &cur); 1757 } 1758 1759 /* 1760 * Add any extra memory after e_data we added to the phys_avail list 1761 * back to the old list. 1762 */ 1763 if (ndata_remain_sz >= MMU_PAGESIZE) 1764 memlist_add(va_to_pa(nalloc_base), 1765 (uint64_t)ndata_remain_sz, &memlist, &cur); 1766 1767 /* 1768 * There isn't any bounds checking on the memlist area 1769 * so ensure it hasn't overgrown. 1770 */ 1771 if ((caddr_t)memlist > (caddr_t)memlist_end) 1772 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1773 1774 /* 1775 * The kernel removes the pages that were allocated for it from 1776 * the freelist, but we now have to find any -extra- pages that 1777 * the prom has allocated for it's own book-keeping, and remove 1778 * them from the freelist too. sigh. 1779 */ 1780 sync_memlists(phys_avail, cur); 1781 1782 ASSERT(phys_avail != NULL); 1783 1784 old_phys_avail = phys_avail; 1785 phys_avail = cur; 1786 } 1787 1788 void 1789 update_kcage_ranges(uint64_t addr, uint64_t len) 1790 { 1791 pfn_t base = btop(addr); 1792 pgcnt_t num = btop(len); 1793 int rv; 1794 1795 rv = kcage_range_add(base, num, kcage_startup_dir); 1796 1797 if (rv == ENOMEM) { 1798 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1799 (len == 0 ? 0 : BYTES_TO_MB(len))); 1800 } else if (rv != 0) { 1801 /* catch this in debug kernels */ 1802 ASSERT(0); 1803 1804 cmn_err(CE_WARN, "unexpected kcage_range_add" 1805 " return value %d", rv); 1806 } 1807 } 1808 1809 static void 1810 startup_vm(void) 1811 { 1812 size_t i; 1813 struct segmap_crargs a; 1814 struct segkpm_crargs b; 1815 1816 uint64_t avmem; 1817 caddr_t va; 1818 pgcnt_t max_phys_segkp; 1819 int mnode; 1820 1821 extern int use_brk_lpg, use_stk_lpg; 1822 1823 /* 1824 * get prom's mappings, create hments for them and switch 1825 * to the kernel context. 1826 */ 1827 hat_kern_setup(); 1828 1829 /* 1830 * Take over trap table 1831 */ 1832 setup_trap_table(); 1833 1834 /* 1835 * Install the va>tte handler, so that the prom can handle 1836 * misses and understand the kernel table layout in case 1837 * we need call into the prom. 1838 */ 1839 install_va_to_tte(); 1840 1841 /* 1842 * Set a flag to indicate that the tba has been taken over. 1843 */ 1844 tba_taken_over = 1; 1845 1846 /* initialize MMU primary context register */ 1847 mmu_init_kcontext(); 1848 1849 /* 1850 * The boot cpu can now take interrupts, x-calls, x-traps 1851 */ 1852 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1853 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1854 1855 /* 1856 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1857 */ 1858 tbr_wr_addr_inited = 1; 1859 1860 /* 1861 * Initialize VM system, and map kernel address space. 1862 */ 1863 kvm_init(); 1864 1865 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1866 if (kernel_cage_enable) { 1867 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1868 } 1869 memlist_free_list(old_phys_avail); 1870 1871 /* 1872 * If the following is true, someone has patched 1873 * phsymem to be less than the number of pages that 1874 * the system actually has. Remove pages until system 1875 * memory is limited to the requested amount. Since we 1876 * have allocated page structures for all pages, we 1877 * correct the amount of memory we want to remove 1878 * by the size of the memory used to hold page structures 1879 * for the non-used pages. 1880 */ 1881 if (physmem + ramdisk_npages < npages) { 1882 pgcnt_t diff, off; 1883 struct page *pp; 1884 struct seg kseg; 1885 1886 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1887 1888 off = 0; 1889 diff = npages - (physmem + ramdisk_npages); 1890 diff -= mmu_btopr(diff * sizeof (struct page)); 1891 kseg.s_as = &kas; 1892 while (diff--) { 1893 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1894 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1895 &kseg, (caddr_t)off); 1896 if (pp == NULL) 1897 cmn_err(CE_PANIC, "limited physmem too much!"); 1898 page_io_unlock(pp); 1899 page_downgrade(pp); 1900 availrmem--; 1901 off += MMU_PAGESIZE; 1902 } 1903 } 1904 1905 /* 1906 * When printing memory, show the total as physmem less 1907 * that stolen by a debugger. 1908 */ 1909 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1910 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1911 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1912 1913 avmem = (uint64_t)freemem << PAGESHIFT; 1914 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1915 1916 /* 1917 * For small memory systems disable automatic large pages. 1918 */ 1919 if (physmem < privm_lpg_min_physmem) { 1920 use_brk_lpg = 0; 1921 use_stk_lpg = 0; 1922 } 1923 1924 /* 1925 * Perform platform specific freelist processing 1926 */ 1927 if (&plat_freelist_process) { 1928 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1929 if (mem_node_config[mnode].exists) 1930 plat_freelist_process(mnode); 1931 } 1932 1933 /* 1934 * Initialize the segkp segment type. We position it 1935 * after the configured tables and buffers (whose end 1936 * is given by econtig) and before V_WKBASE_ADDR. 1937 * Also in this area is segkmap (size SEGMAPSIZE). 1938 */ 1939 1940 /* XXX - cache alignment? */ 1941 va = (caddr_t)SEGKPBASE; 1942 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1943 1944 max_phys_segkp = (physmem * 2); 1945 1946 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1947 segkpsize = btop(SEGKPDEFSIZE); 1948 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1949 "segkpsize has been reset to %ld pages", segkpsize); 1950 } 1951 1952 i = ptob(MIN(segkpsize, max_phys_segkp)); 1953 1954 rw_enter(&kas.a_lock, RW_WRITER); 1955 if (seg_attach(&kas, va, i, segkp) < 0) 1956 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1957 if (segkp_create(segkp) != 0) 1958 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1959 rw_exit(&kas.a_lock); 1960 1961 /* 1962 * kpm segment 1963 */ 1964 segmap_kpm = kpm_enable && 1965 segmap_kpm && PAGESIZE == MAXBSIZE; 1966 1967 if (kpm_enable) { 1968 rw_enter(&kas.a_lock, RW_WRITER); 1969 1970 /* 1971 * The segkpm virtual range range is larger than the 1972 * actual physical memory size and also covers gaps in 1973 * the physical address range for the following reasons: 1974 * . keep conversion between segkpm and physical addresses 1975 * simple, cheap and unambiguous. 1976 * . avoid extension/shrink of the the segkpm in case of DR. 1977 * . avoid complexity for handling of virtual addressed 1978 * caches, segkpm and the regular mapping scheme must be 1979 * kept in sync wrt. the virtual color of mapped pages. 1980 * Any accesses to virtual segkpm ranges not backed by 1981 * physical memory will fall through the memseg pfn hash 1982 * and will be handled in segkpm_fault. 1983 * Additional kpm_size spaces needed for vac alias prevention. 1984 */ 1985 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1986 segkpm) < 0) 1987 cmn_err(CE_PANIC, "cannot attach segkpm"); 1988 1989 b.prot = PROT_READ | PROT_WRITE; 1990 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1991 1992 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1993 panic("segkpm_create segkpm"); 1994 1995 rw_exit(&kas.a_lock); 1996 1997 mach_kpm_init(); 1998 } 1999 2000 va = kpm_vbase + (kpm_size * vac_colors); 2001 2002 if (!segzio_fromheap) { 2003 size_t size; 2004 size_t physmem_b = mmu_ptob(physmem); 2005 2006 /* size is in bytes, segziosize is in pages */ 2007 if (segziosize == 0) { 2008 size = physmem_b; 2009 } else { 2010 size = mmu_ptob(segziosize); 2011 } 2012 2013 if (size < SEGZIOMINSIZE) { 2014 size = SEGZIOMINSIZE; 2015 } else if (size > SEGZIOMAXSIZE) { 2016 size = SEGZIOMAXSIZE; 2017 /* 2018 * On 64-bit x86, we only have 2TB of KVA. This exists 2019 * for parity with x86. 2020 * 2021 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2022 * doesn't consume all of KVA. However, if we have a 2023 * system that has more thant 512gb of physical memory, 2024 * we can actually consume about half of the difference 2025 * between 512gb and the rest of the available physical 2026 * memory. 2027 */ 2028 if (physmem_b > SEGZIOMAXSIZE) { 2029 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2030 } 2031 } 2032 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2033 /* put the base of the ZIO segment after the kpm segment */ 2034 segzio_base = va; 2035 va += mmu_ptob(segziosize); 2036 PRM_DEBUG(segziosize); 2037 PRM_DEBUG(segzio_base); 2038 2039 /* 2040 * On some platforms, kvm_init is called after the kpm 2041 * sizes have been determined. On SPARC, kvm_init is called 2042 * before, so we have to attach the kzioseg after kvm is 2043 * initialized, otherwise we'll try to allocate from the boot 2044 * area since the kernel heap hasn't yet been configured. 2045 */ 2046 rw_enter(&kas.a_lock, RW_WRITER); 2047 2048 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2049 &kzioseg); 2050 (void) segkmem_zio_create(&kzioseg); 2051 2052 /* create zio area covering new segment */ 2053 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2054 2055 rw_exit(&kas.a_lock); 2056 } 2057 2058 if (ppvm_enable) { 2059 caddr_t ppvm_max; 2060 2061 /* 2062 * ppvm refers to the static VA space used to map 2063 * the page_t's for dynamically added memory. 2064 * 2065 * ppvm_base should not cross a potential VA hole. 2066 * 2067 * ppvm_size should be large enough to map the 2068 * page_t's needed to manage all of KPM range. 2069 */ 2070 ppvm_size = 2071 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2072 MMU_PAGESIZE); 2073 ppvm_max = (caddr_t)(0ull - ppvm_size); 2074 ppvm_base = (page_t *)va; 2075 2076 if ((caddr_t)ppvm_base <= hole_end) { 2077 cmn_err(CE_WARN, 2078 "Memory DR disabled: invalid DR map base: 0x%p\n", 2079 (void *)ppvm_base); 2080 ppvm_enable = 0; 2081 } else if ((caddr_t)ppvm_base > ppvm_max) { 2082 uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2083 2084 cmn_err(CE_WARN, 2085 "Memory DR disabled: insufficient DR map size:" 2086 " 0x%lx (needed 0x%lx)\n", 2087 ppvm_size - diff, ppvm_size); 2088 ppvm_enable = 0; 2089 } 2090 PRM_DEBUG(ppvm_size); 2091 PRM_DEBUG(ppvm_base); 2092 } 2093 2094 /* 2095 * Now create generic mapping segment. This mapping 2096 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2097 * virtual address is greater than the amount of free 2098 * memory that is available, then we trim back the 2099 * segment size to that amount 2100 */ 2101 va = (caddr_t)SEGMAPBASE; 2102 2103 /* 2104 * 1201049: segkmap base address must be MAXBSIZE aligned 2105 */ 2106 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2107 2108 /* 2109 * Set size of segmap to percentage of freemem at boot, 2110 * but stay within the allowable range 2111 * Note we take percentage before converting from pages 2112 * to bytes to avoid an overflow on 32-bit kernels. 2113 */ 2114 i = mmu_ptob((freemem * segmap_percent) / 100); 2115 2116 if (i < MINMAPSIZE) 2117 i = MINMAPSIZE; 2118 2119 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2120 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2121 2122 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2123 2124 rw_enter(&kas.a_lock, RW_WRITER); 2125 if (seg_attach(&kas, va, i, segkmap) < 0) 2126 cmn_err(CE_PANIC, "cannot attach segkmap"); 2127 2128 a.prot = PROT_READ | PROT_WRITE; 2129 a.shmsize = shm_alignment; 2130 a.nfreelist = 0; /* use segmap driver defaults */ 2131 2132 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2133 panic("segmap_create segkmap"); 2134 rw_exit(&kas.a_lock); 2135 2136 segdev_init(); 2137 } 2138 2139 static void 2140 startup_end(void) 2141 { 2142 if ((caddr_t)memlist > (caddr_t)memlist_end) 2143 panic("memlist overflow 2"); 2144 memlist_free_block((caddr_t)memlist, 2145 ((caddr_t)memlist_end - (caddr_t)memlist)); 2146 memlist = NULL; 2147 2148 /* enable page_relocation since OBP is now done */ 2149 page_relocate_ready = 1; 2150 2151 /* 2152 * Perform tasks that get done after most of the VM 2153 * initialization has been done but before the clock 2154 * and other devices get started. 2155 */ 2156 kern_setup1(); 2157 2158 /* 2159 * Perform CPC initialization for this CPU. 2160 */ 2161 kcpc_hw_init(); 2162 2163 /* 2164 * Intialize the VM arenas for allocating physically 2165 * contiguus memory chunk for interrupt queues snd 2166 * allocate/register boot cpu's queues, if any and 2167 * allocate dump buffer for sun4v systems to store 2168 * extra crash information during crash dump 2169 */ 2170 contig_mem_init(); 2171 mach_descrip_init(); 2172 2173 if (cpu_intrq_setup(CPU)) { 2174 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2175 } 2176 cpu_intrq_register(CPU); 2177 mach_htraptrace_setup(CPU->cpu_id); 2178 mach_htraptrace_configure(CPU->cpu_id); 2179 mach_dump_buffer_init(); 2180 2181 /* 2182 * Initialize interrupt related stuff 2183 */ 2184 cpu_intr_alloc(CPU, NINTR_THREADS); 2185 2186 (void) splzs(); /* allow hi clock ints but not zs */ 2187 2188 /* 2189 * Initialize errors. 2190 */ 2191 error_init(); 2192 2193 /* 2194 * Note that we may have already used kernel bcopy before this 2195 * point - but if you really care about this, adb the use_hw_* 2196 * variables to 0 before rebooting. 2197 */ 2198 mach_hw_copy_limit(); 2199 2200 /* 2201 * Install the "real" preemption guards before DDI services 2202 * are available. 2203 */ 2204 (void) prom_set_preprom(kern_preprom); 2205 (void) prom_set_postprom(kern_postprom); 2206 CPU->cpu_m.mutex_ready = 1; 2207 2208 /* 2209 * Initialize segnf (kernel support for non-faulting loads). 2210 */ 2211 segnf_init(); 2212 2213 /* 2214 * Configure the root devinfo node. 2215 */ 2216 configure(); /* set up devices */ 2217 mach_cpu_halt_idle(); 2218 } 2219 2220 2221 void 2222 post_startup(void) 2223 { 2224 #ifdef PTL1_PANIC_DEBUG 2225 extern void init_ptl1_thread(void); 2226 #endif /* PTL1_PANIC_DEBUG */ 2227 extern void abort_sequence_init(void); 2228 2229 /* 2230 * Set the system wide, processor-specific flags to be passed 2231 * to userland via the aux vector for performance hints and 2232 * instruction set extensions. 2233 */ 2234 bind_hwcap(); 2235 2236 /* 2237 * Startup memory scrubber (if any) 2238 */ 2239 mach_memscrub(); 2240 2241 /* 2242 * Allocate soft interrupt to handle abort sequence. 2243 */ 2244 abort_sequence_init(); 2245 2246 /* 2247 * Configure the rest of the system. 2248 * Perform forceloading tasks for /etc/system. 2249 */ 2250 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2251 /* 2252 * ON4.0: Force /proc module in until clock interrupt handle fixed 2253 * ON4.0: This must be fixed or restated in /etc/systems. 2254 */ 2255 (void) modload("fs", "procfs"); 2256 2257 /* load machine class specific drivers */ 2258 load_mach_drivers(); 2259 2260 /* load platform specific drivers */ 2261 if (&load_platform_drivers) 2262 load_platform_drivers(); 2263 2264 /* load vis simulation module, if we are running w/fpu off */ 2265 if (!fpu_exists) { 2266 if (modload("misc", "vis") == -1) 2267 halt("Can't load vis"); 2268 } 2269 2270 mach_fpras(); 2271 2272 maxmem = freemem; 2273 2274 pg_init(); 2275 2276 #ifdef PTL1_PANIC_DEBUG 2277 init_ptl1_thread(); 2278 #endif /* PTL1_PANIC_DEBUG */ 2279 } 2280 2281 #ifdef PTL1_PANIC_DEBUG 2282 int ptl1_panic_test = 0; 2283 int ptl1_panic_xc_one_test = 0; 2284 int ptl1_panic_xc_all_test = 0; 2285 int ptl1_panic_xt_one_test = 0; 2286 int ptl1_panic_xt_all_test = 0; 2287 kthread_id_t ptl1_thread_p = NULL; 2288 kcondvar_t ptl1_cv; 2289 kmutex_t ptl1_mutex; 2290 int ptl1_recurse_count_threshold = 0x40; 2291 int ptl1_recurse_trap_threshold = 0x3d; 2292 extern void ptl1_recurse(int, int); 2293 extern void ptl1_panic_xt(int, int); 2294 2295 /* 2296 * Called once per second by timeout() to wake up 2297 * the ptl1_panic thread to see if it should cause 2298 * a trap to the ptl1_panic() code. 2299 */ 2300 /* ARGSUSED */ 2301 static void 2302 ptl1_wakeup(void *arg) 2303 { 2304 mutex_enter(&ptl1_mutex); 2305 cv_signal(&ptl1_cv); 2306 mutex_exit(&ptl1_mutex); 2307 } 2308 2309 /* 2310 * ptl1_panic cross call function: 2311 * Needed because xc_one() and xc_some() can pass 2312 * 64 bit args but ptl1_recurse() expects ints. 2313 */ 2314 static void 2315 ptl1_panic_xc(void) 2316 { 2317 ptl1_recurse(ptl1_recurse_count_threshold, 2318 ptl1_recurse_trap_threshold); 2319 } 2320 2321 /* 2322 * The ptl1 thread waits for a global flag to be set 2323 * and uses the recurse thresholds to set the stack depth 2324 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2325 * or indirectly via the cross call and cross trap functions. 2326 * 2327 * This is useful testing stack overflows and normal 2328 * ptl1_panic() states with a know stack frame. 2329 * 2330 * ptl1_recurse() is an asm function in ptl1_panic.s that 2331 * sets the {In, Local, Out, and Global} registers to a 2332 * know state on the stack and just prior to causing a 2333 * test ptl1_panic trap. 2334 */ 2335 static void 2336 ptl1_thread(void) 2337 { 2338 mutex_enter(&ptl1_mutex); 2339 while (ptl1_thread_p) { 2340 cpuset_t other_cpus; 2341 int cpu_id; 2342 int my_cpu_id; 2343 int target_cpu_id; 2344 int target_found; 2345 2346 if (ptl1_panic_test) { 2347 ptl1_recurse(ptl1_recurse_count_threshold, 2348 ptl1_recurse_trap_threshold); 2349 } 2350 2351 /* 2352 * Find potential targets for x-call and x-trap, 2353 * if any exist while preempt is disabled we 2354 * start a ptl1_panic if requested via a 2355 * globals. 2356 */ 2357 kpreempt_disable(); 2358 my_cpu_id = CPU->cpu_id; 2359 other_cpus = cpu_ready_set; 2360 CPUSET_DEL(other_cpus, CPU->cpu_id); 2361 target_found = 0; 2362 if (!CPUSET_ISNULL(other_cpus)) { 2363 /* 2364 * Pick the first one 2365 */ 2366 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2367 if (cpu_id == my_cpu_id) 2368 continue; 2369 2370 if (CPU_XCALL_READY(cpu_id)) { 2371 target_cpu_id = cpu_id; 2372 target_found = 1; 2373 break; 2374 } 2375 } 2376 ASSERT(target_found); 2377 2378 if (ptl1_panic_xc_one_test) { 2379 xc_one(target_cpu_id, 2380 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2381 } 2382 if (ptl1_panic_xc_all_test) { 2383 xc_some(other_cpus, 2384 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2385 } 2386 if (ptl1_panic_xt_one_test) { 2387 xt_one(target_cpu_id, 2388 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2389 } 2390 if (ptl1_panic_xt_all_test) { 2391 xt_some(other_cpus, 2392 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2393 } 2394 } 2395 kpreempt_enable(); 2396 (void) timeout(ptl1_wakeup, NULL, hz); 2397 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2398 } 2399 mutex_exit(&ptl1_mutex); 2400 } 2401 2402 /* 2403 * Called during early startup to create the ptl1_thread 2404 */ 2405 void 2406 init_ptl1_thread(void) 2407 { 2408 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2409 &p0, TS_RUN, 0); 2410 } 2411 #endif /* PTL1_PANIC_DEBUG */ 2412 2413 2414 static void 2415 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2416 { 2417 struct memlist *new; 2418 2419 new = *memlistp; 2420 new->address = start; 2421 new->size = len; 2422 *memlistp = new + 1; 2423 } 2424 2425 /* 2426 * Add to a memory list. 2427 * start = start of new memory segment 2428 * len = length of new memory segment in bytes 2429 * memlistp = pointer to array of available memory segment structures 2430 * curmemlistp = memory list to which to add segment. 2431 */ 2432 static void 2433 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2434 struct memlist **curmemlistp) 2435 { 2436 struct memlist *new = *memlistp; 2437 2438 memlist_new(start, len, memlistp); 2439 memlist_insert(new, curmemlistp); 2440 } 2441 2442 static int 2443 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2444 { 2445 int nseg; 2446 size_t memseg_sz; 2447 struct memseg *msp; 2448 2449 /* 2450 * The memseg list is for the chunks of physical memory that 2451 * will be managed by the vm system. The number calculated is 2452 * a guess as boot may fragment it more when memory allocations 2453 * are made before kphysm_init(). 2454 */ 2455 memseg_sz = (avail + 10) * sizeof (struct memseg); 2456 memseg_sz = roundup(memseg_sz, PAGESIZE); 2457 nseg = memseg_sz / sizeof (struct memseg); 2458 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2459 if (msp == NULL) 2460 return (1); 2461 PRM_DEBUG(memseg_free); 2462 2463 while (nseg--) { 2464 msp->next = memseg_free; 2465 memseg_free = msp; 2466 msp++; 2467 } 2468 return (0); 2469 } 2470 2471 /* 2472 * In the case of architectures that support dynamic addition of 2473 * memory at run-time there are two cases where memsegs need to 2474 * be initialized and added to the memseg list. 2475 * 1) memsegs that are constructed at startup. 2476 * 2) memsegs that are constructed at run-time on 2477 * hot-plug capable architectures. 2478 * This code was originally part of the function kphysm_init(). 2479 */ 2480 2481 static void 2482 memseg_list_add(struct memseg *memsegp) 2483 { 2484 struct memseg **prev_memsegp; 2485 pgcnt_t num; 2486 2487 /* insert in memseg list, decreasing number of pages order */ 2488 2489 num = MSEG_NPAGES(memsegp); 2490 2491 for (prev_memsegp = &memsegs; *prev_memsegp; 2492 prev_memsegp = &((*prev_memsegp)->next)) { 2493 if (num > MSEG_NPAGES(*prev_memsegp)) 2494 break; 2495 } 2496 2497 memsegp->next = *prev_memsegp; 2498 *prev_memsegp = memsegp; 2499 2500 if (kpm_enable) { 2501 memsegp->nextpa = (memsegp->next) ? 2502 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2503 2504 if (prev_memsegp != &memsegs) { 2505 struct memseg *msp; 2506 msp = (struct memseg *)((caddr_t)prev_memsegp - 2507 offsetof(struct memseg, next)); 2508 msp->nextpa = va_to_pa(memsegp); 2509 } else { 2510 memsegspa = va_to_pa(memsegs); 2511 } 2512 } 2513 } 2514 2515 /* 2516 * PSM add_physmem_cb(). US-II and newer processors have some 2517 * flavor of the prefetch capability implemented. We exploit 2518 * this capability for optimum performance. 2519 */ 2520 #define PREFETCH_BYTES 64 2521 2522 void 2523 add_physmem_cb(page_t *pp, pfn_t pnum) 2524 { 2525 extern void prefetch_page_w(void *); 2526 2527 pp->p_pagenum = pnum; 2528 2529 /* 2530 * Prefetch one more page_t into E$. To prevent future 2531 * mishaps with the sizeof(page_t) changing on us, we 2532 * catch this on debug kernels if we can't bring in the 2533 * entire hpage with 2 PREFETCH_BYTES reads. See 2534 * also, sun4u/cpu/cpu_module.c 2535 */ 2536 /*LINTED*/ 2537 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2538 prefetch_page_w((char *)pp); 2539 } 2540 2541 /* 2542 * Find memseg with given pfn 2543 */ 2544 static struct memseg * 2545 memseg_find(pfn_t base, pfn_t *next) 2546 { 2547 struct memseg *seg; 2548 2549 if (next != NULL) 2550 *next = LONG_MAX; 2551 for (seg = memsegs; seg != NULL; seg = seg->next) { 2552 if (base >= seg->pages_base && base < seg->pages_end) 2553 return (seg); 2554 if (next != NULL && seg->pages_base > base && 2555 seg->pages_base < *next) 2556 *next = seg->pages_base; 2557 } 2558 return (NULL); 2559 } 2560 2561 extern struct vnode prom_ppages; 2562 2563 /* 2564 * Put page allocated by OBP on prom_ppages 2565 */ 2566 static void 2567 kphysm_erase(uint64_t addr, uint64_t len) 2568 { 2569 struct page *pp; 2570 struct memseg *seg; 2571 pfn_t base = btop(addr), next; 2572 pgcnt_t num = btop(len); 2573 2574 while (num != 0) { 2575 pgcnt_t off, left; 2576 2577 seg = memseg_find(base, &next); 2578 if (seg == NULL) { 2579 if (next == LONG_MAX) 2580 break; 2581 left = MIN(next - base, num); 2582 base += left, num -= left; 2583 continue; 2584 } 2585 off = base - seg->pages_base; 2586 pp = seg->pages + off; 2587 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2588 while (num != left) { 2589 /* 2590 * init it, lock it, and hashin on prom_pages vp. 2591 * 2592 * Mark it as NONRELOC to let DR know the page 2593 * is locked long term, otherwise DR hangs when 2594 * trying to remove those pages. 2595 * 2596 * XXX vnode offsets on the prom_ppages vnode 2597 * are page numbers (gack) for >32 bit 2598 * physical memory machines. 2599 */ 2600 PP_SETNORELOC(pp); 2601 add_physmem_cb(pp, base); 2602 if (page_trylock(pp, SE_EXCL) == 0) 2603 cmn_err(CE_PANIC, "prom page locked"); 2604 (void) page_hashin(pp, &prom_ppages, 2605 (offset_t)base, NULL); 2606 (void) page_pp_lock(pp, 0, 1); 2607 pp++, base++, num--; 2608 } 2609 } 2610 } 2611 2612 static page_t *ppnext; 2613 static pgcnt_t ppleft; 2614 2615 static void *kpm_ppnext; 2616 static pgcnt_t kpm_ppleft; 2617 2618 /* 2619 * Create a memseg 2620 */ 2621 static void 2622 kphysm_memseg(uint64_t addr, uint64_t len) 2623 { 2624 pfn_t base = btop(addr); 2625 pgcnt_t num = btop(len); 2626 struct memseg *seg; 2627 2628 seg = memseg_free; 2629 memseg_free = seg->next; 2630 ASSERT(seg != NULL); 2631 2632 seg->pages = ppnext; 2633 seg->epages = ppnext + num; 2634 seg->pages_base = base; 2635 seg->pages_end = base + num; 2636 ppnext += num; 2637 ppleft -= num; 2638 2639 if (kpm_enable) { 2640 pgcnt_t kpnum = ptokpmpr(num); 2641 2642 if (kpnum > kpm_ppleft) 2643 panic("kphysm_memseg: kpm_pp overflow"); 2644 seg->pagespa = va_to_pa(seg->pages); 2645 seg->epagespa = va_to_pa(seg->epages); 2646 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2647 seg->kpm_nkpmpgs = kpnum; 2648 /* 2649 * In the kpm_smallpage case, the kpm array 2650 * is 1-1 wrt the page array 2651 */ 2652 if (kpm_smallpages) { 2653 kpm_spage_t *kpm_pp = kpm_ppnext; 2654 2655 kpm_ppnext = kpm_pp + kpnum; 2656 seg->kpm_spages = kpm_pp; 2657 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2658 } else { 2659 kpm_page_t *kpm_pp = kpm_ppnext; 2660 2661 kpm_ppnext = kpm_pp + kpnum; 2662 seg->kpm_pages = kpm_pp; 2663 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2664 /* ASSERT no kpm overlaps */ 2665 ASSERT( 2666 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2667 ASSERT(memseg_find( 2668 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2669 } 2670 kpm_ppleft -= kpnum; 2671 } 2672 2673 memseg_list_add(seg); 2674 } 2675 2676 /* 2677 * Add range to free list 2678 */ 2679 void 2680 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2681 { 2682 struct page *pp; 2683 struct memseg *seg; 2684 pfn_t base = btop(addr); 2685 pgcnt_t num = btop(len); 2686 2687 seg = memseg_find(base, NULL); 2688 ASSERT(seg != NULL); 2689 pp = seg->pages + (base - seg->pages_base); 2690 2691 if (reclaim) { 2692 struct page *rpp = pp; 2693 struct page *lpp = pp + num; 2694 2695 /* 2696 * page should be locked on prom_ppages 2697 * unhash and unlock it 2698 */ 2699 while (rpp < lpp) { 2700 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages); 2701 ASSERT(PP_ISNORELOC(rpp)); 2702 PP_CLRNORELOC(rpp); 2703 page_pp_unlock(rpp, 0, 1); 2704 page_hashout(rpp, NULL); 2705 page_unlock(rpp); 2706 rpp++; 2707 } 2708 } 2709 2710 /* 2711 * add_physmem() initializes the PSM part of the page 2712 * struct by calling the PSM back with add_physmem_cb(). 2713 * In addition it coalesces pages into larger pages as 2714 * it initializes them. 2715 */ 2716 add_physmem(pp, num, base); 2717 } 2718 2719 /* 2720 * kphysm_init() tackles the problem of initializing physical memory. 2721 */ 2722 static void 2723 kphysm_init(void) 2724 { 2725 struct memlist *pmem; 2726 2727 ASSERT(page_hash != NULL && page_hashsz != 0); 2728 2729 ppnext = pp_base; 2730 ppleft = npages; 2731 kpm_ppnext = kpm_pp_base; 2732 kpm_ppleft = kpm_npages; 2733 2734 /* 2735 * installed pages not on nopp_memlist go in memseg list 2736 */ 2737 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2738 2739 /* 2740 * Free the avail list 2741 */ 2742 for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 2743 kphysm_add(pmem->address, pmem->size, 0); 2744 2745 /* 2746 * Erase pages that aren't available 2747 */ 2748 diff_memlists(phys_install, phys_avail, kphysm_erase); 2749 2750 build_pfn_hash(); 2751 } 2752 2753 /* 2754 * Kernel VM initialization. 2755 * Assumptions about kernel address space ordering: 2756 * (1) gap (user space) 2757 * (2) kernel text 2758 * (3) kernel data/bss 2759 * (4) gap 2760 * (5) kernel data structures 2761 * (6) gap 2762 * (7) debugger (optional) 2763 * (8) monitor 2764 * (9) gap (possibly null) 2765 * (10) dvma 2766 * (11) devices 2767 */ 2768 static void 2769 kvm_init(void) 2770 { 2771 /* 2772 * Put the kernel segments in kernel address space. 2773 */ 2774 rw_enter(&kas.a_lock, RW_WRITER); 2775 as_avlinit(&kas); 2776 2777 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2778 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2779 (void) segkmem_create(&ktextseg); 2780 2781 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2782 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2783 (void) segkmem_create(&ktexthole); 2784 2785 (void) seg_attach(&kas, (caddr_t)valloc_base, 2786 (size_t)(econtig32 - valloc_base), &kvalloc); 2787 (void) segkmem_create(&kvalloc); 2788 2789 if (kmem64_base) { 2790 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2791 (size_t)(kmem64_end - kmem64_base), &kmem64); 2792 (void) segkmem_create(&kmem64); 2793 } 2794 2795 /* 2796 * We're about to map out /boot. This is the beginning of the 2797 * system resource management transition. We can no longer 2798 * call into /boot for I/O or memory allocations. 2799 */ 2800 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2801 (void) segkmem_create(&kvseg); 2802 hblk_alloc_dynamic = 1; 2803 2804 /* 2805 * we need to preallocate pages for DR operations before enabling large 2806 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2807 */ 2808 memseg_remap_init(); 2809 2810 /* at this point we are ready to use large page heap */ 2811 segkmem_heap_lp_init(); 2812 2813 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2814 &kvseg32); 2815 (void) segkmem_create(&kvseg32); 2816 2817 /* 2818 * Create a segment for the debugger. 2819 */ 2820 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2821 (void) segkmem_create(&kdebugseg); 2822 2823 rw_exit(&kas.a_lock); 2824 } 2825 2826 char obp_tte_str[] = 2827 "h# %x constant MMU_PAGESHIFT " 2828 "h# %x constant TTE8K " 2829 "h# %x constant SFHME_SIZE " 2830 "h# %x constant SFHME_TTE " 2831 "h# %x constant HMEBLK_TAG " 2832 "h# %x constant HMEBLK_NEXT " 2833 "h# %x constant HMEBLK_MISC " 2834 "h# %x constant HMEBLK_HME1 " 2835 "h# %x constant NHMENTS " 2836 "h# %x constant HBLK_SZMASK " 2837 "h# %x constant HBLK_RANGE_SHIFT " 2838 "h# %x constant HMEBP_HBLK " 2839 "h# %x constant HMEBLK_ENDPA " 2840 "h# %x constant HMEBUCKET_SIZE " 2841 "h# %x constant HTAG_SFMMUPSZ " 2842 "h# %x constant HTAG_BSPAGE_SHIFT " 2843 "h# %x constant HTAG_REHASH_SHIFT " 2844 "h# %x constant SFMMU_INVALID_SHMERID " 2845 "h# %x constant mmu_hashcnt " 2846 "h# %p constant uhme_hash " 2847 "h# %p constant khme_hash " 2848 "h# %x constant UHMEHASH_SZ " 2849 "h# %x constant KHMEHASH_SZ " 2850 "h# %p constant KCONTEXT " 2851 "h# %p constant KHATID " 2852 "h# %x constant ASI_MEM " 2853 2854 ": PHYS-X@ ( phys -- data ) " 2855 " ASI_MEM spacex@ " 2856 "; " 2857 2858 ": PHYS-W@ ( phys -- data ) " 2859 " ASI_MEM spacew@ " 2860 "; " 2861 2862 ": PHYS-L@ ( phys -- data ) " 2863 " ASI_MEM spaceL@ " 2864 "; " 2865 2866 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2867 " 3 * MMU_PAGESHIFT + " 2868 "; " 2869 2870 ": TTE_IS_VALID ( ttep -- flag ) " 2871 " PHYS-X@ 0< " 2872 "; " 2873 2874 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2875 " dup TTE8K = if " 2876 " drop HBLK_RANGE_SHIFT " 2877 " else " 2878 " TTE_PAGE_SHIFT " 2879 " then " 2880 "; " 2881 2882 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2883 " tuck >> swap MMU_PAGESHIFT - << " 2884 "; " 2885 2886 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2887 " >> over xor swap ( hash sfmmup ) " 2888 " KHATID <> if ( hash ) " 2889 " UHMEHASH_SZ and ( bucket ) " 2890 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2891 " else ( hash ) " 2892 " KHMEHASH_SZ and ( bucket ) " 2893 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2894 " then ( hmebp ) " 2895 "; " 2896 2897 ": HME_HASH_TABLE_SEARCH " 2898 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2899 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2900 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2901 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2902 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2903 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2904 " else " 2905 " hmeblk_next + phys-x@ false " 2906 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2907 " then " 2908 " else " 2909 " hmeblk_next + phys-x@ false " 2910 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2911 " then " 2912 " else " 2913 " drop 0 true " 2914 " then " 2915 " until r> drop " 2916 "; " 2917 2918 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2919 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2920 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2921 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2922 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2923 "; " 2924 2925 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2926 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2927 " TTE8K = if ( hmeblkp addr ) " 2928 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2929 " else ( hmeblkp addr ) " 2930 " drop 0 ( hmeblkp 0 ) " 2931 " then ( hmeblkp hme-index ) " 2932 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2933 " SFHME_TTE + ( ttep ) " 2934 "; " 2935 2936 ": unix-tte ( addr cnum -- false | tte-data true ) " 2937 " KCONTEXT = if ( addr ) " 2938 " KHATID ( addr khatid ) " 2939 " else ( addr ) " 2940 " drop false exit ( false ) " 2941 " then " 2942 " ( addr khatid ) " 2943 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2944 " 2dup swap i HME_HASH_SHIFT " 2945 "( addr sfmmup sfmmup addr hmeshift ) " 2946 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2947 " over i 4 pick " 2948 "( addr sfmmup hmebp sfmmup rehash addr ) " 2949 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2950 " HME_HASH_TABLE_SEARCH " 2951 "( addr sfmmup { null | hmeblkp } ) " 2952 " ?dup if ( addr sfmmup hmeblkp ) " 2953 " nip swap HBLK_TO_TTEP ( ttep ) " 2954 " dup TTE_IS_VALID if ( valid-ttep ) " 2955 " PHYS-X@ true ( tte-data true ) " 2956 " else ( invalid-tte ) " 2957 " drop false ( false ) " 2958 " then ( false | tte-data true ) " 2959 " unloop exit ( false | tte-data true ) " 2960 " then ( addr sfmmup ) " 2961 " loop ( addr sfmmup ) " 2962 " 2drop false ( false ) " 2963 "; " 2964 ; 2965 2966 void 2967 create_va_to_tte(void) 2968 { 2969 char *bp; 2970 extern int khmehash_num, uhmehash_num; 2971 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2972 2973 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2974 2975 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2976 2977 /* 2978 * Teach obp how to parse our sw ttes. 2979 */ 2980 (void) sprintf(bp, obp_tte_str, 2981 MMU_PAGESHIFT, 2982 TTE8K, 2983 sizeof (struct sf_hment), 2984 OFFSET(struct sf_hment, hme_tte), 2985 OFFSET(struct hme_blk, hblk_tag), 2986 OFFSET(struct hme_blk, hblk_nextpa), 2987 OFFSET(struct hme_blk, hblk_misc), 2988 OFFSET(struct hme_blk, hblk_hme), 2989 NHMENTS, 2990 HBLK_SZMASK, 2991 HBLK_RANGE_SHIFT, 2992 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2993 HMEBLK_ENDPA, 2994 sizeof (struct hmehash_bucket), 2995 HTAG_SFMMUPSZ, 2996 HTAG_BSPAGE_SHIFT, 2997 HTAG_REHASH_SHIFT, 2998 SFMMU_INVALID_SHMERID, 2999 mmu_hashcnt, 3000 (caddr_t)va_to_pa((caddr_t)uhme_hash), 3001 (caddr_t)va_to_pa((caddr_t)khme_hash), 3002 UHMEHASH_SZ, 3003 KHMEHASH_SZ, 3004 KCONTEXT, 3005 KHATID, 3006 ASI_MEM); 3007 prom_interpret(bp, 0, 0, 0, 0, 0); 3008 3009 kobj_free(bp, MMU_PAGESIZE); 3010 } 3011 3012 void 3013 install_va_to_tte(void) 3014 { 3015 /* 3016 * advise prom that he can use unix-tte 3017 */ 3018 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3019 } 3020 3021 /* 3022 * Here we add "device-type=console" for /os-io node, for currently 3023 * our kernel console output only supports displaying text and 3024 * performing cursor-positioning operations (through kernel framebuffer 3025 * driver) and it doesn't support other functionalities required for a 3026 * standard "display" device as specified in 1275 spec. The main missing 3027 * interface defined by the 1275 spec is "draw-logo". 3028 * also see the comments above prom_stdout_is_framebuffer(). 3029 */ 3030 static char *create_node = 3031 "\" /\" find-device " 3032 "new-device " 3033 "\" os-io\" device-name " 3034 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3035 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3036 " 2>r swap 2 2r> ['] $callback catch if " 3037 " 2drop 3drop 0 " 3038 " then " 3039 "; " 3040 ": read ( adr,len -- #read ) " 3041 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3042 " ( retN ... ret1 N ) " 3043 " ?dup if " 3044 " swap >r 1- 0 ?do drop loop r> " 3045 " else " 3046 " -2 " 3047 " then " 3048 "; " 3049 ": write ( adr,len -- #written ) " 3050 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3051 " ( retN ... ret1 N ) " 3052 " ?dup if " 3053 " swap >r 1- 0 ?do drop loop r> " 3054 " else " 3055 " 0 " 3056 " then " 3057 "; " 3058 ": poll-tty ( -- ) ; " 3059 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3060 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3061 ": cb-give/take ( $method -- ) " 3062 " 0 -rot ['] $callback catch ?dup if " 3063 " >r 2drop 2drop r> throw " 3064 " else " 3065 " 0 ?do drop loop " 3066 " then " 3067 "; " 3068 ": give ( -- ) \" exit-input\" cb-give/take ; " 3069 ": take ( -- ) \" enter-input\" cb-give/take ; " 3070 ": open ( -- ok? ) true ; " 3071 ": close ( -- ) ; " 3072 "finish-device " 3073 "device-end "; 3074 3075 /* 3076 * Create the OBP input/output node (FCode serial driver). 3077 * It is needed for both USB console keyboard and for 3078 * the kernel terminal emulator. It is too early to check for a 3079 * kernel console compatible framebuffer now, so we create this 3080 * so that we're ready if we need to enable kernel terminal emulation. 3081 * 3082 * When the USB software takes over the input device at the time 3083 * consconfig runs, OBP's stdin is redirected to this node. 3084 * Whenever the FORTH user interface is used after this switch, 3085 * the node will call back into the kernel for console input. 3086 * If a serial device such as ttya or a UART with a Type 5 keyboard 3087 * attached is used, OBP takes over the serial device when the system 3088 * goes to the debugger after the system is booted. This sharing 3089 * of the relatively simple serial device is difficult but possible. 3090 * Sharing the USB host controller is impossible due its complexity. 3091 * 3092 * Similarly to USB keyboard input redirection, after consconfig_dacf 3093 * configures a kernel console framebuffer as the standard output 3094 * device, OBP's stdout is switched to to vector through the 3095 * /os-io node into the kernel terminal emulator. 3096 */ 3097 static void 3098 startup_create_io_node(void) 3099 { 3100 prom_interpret(create_node, 0, 0, 0, 0, 0); 3101 } 3102 3103 3104 static void 3105 do_prom_version_check(void) 3106 { 3107 int i; 3108 pnode_t node; 3109 char buf[64]; 3110 static char drev[] = "Down-rev firmware detected%s\n" 3111 "\tPlease upgrade to the following minimum version:\n" 3112 "\t\t%s\n"; 3113 3114 i = prom_version_check(buf, sizeof (buf), &node); 3115 3116 if (i == PROM_VER64_OK) 3117 return; 3118 3119 if (i == PROM_VER64_UPGRADE) { 3120 cmn_err(CE_WARN, drev, "", buf); 3121 3122 #ifdef DEBUG 3123 prom_enter_mon(); /* Type 'go' to continue */ 3124 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3125 return; 3126 #else 3127 halt(0); 3128 #endif 3129 } 3130 3131 /* 3132 * The other possibility is that this is a server running 3133 * good firmware, but down-rev firmware was detected on at 3134 * least one other cpu board. We just complain if we see 3135 * that. 3136 */ 3137 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3138 } 3139 3140 3141 /* 3142 * Must be defined in platform dependent code. 3143 */ 3144 extern caddr_t modtext; 3145 extern size_t modtext_sz; 3146 extern caddr_t moddata; 3147 3148 #define HEAPTEXT_ARENA(addr) \ 3149 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3150 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3151 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3152 3153 #define HEAPTEXT_OVERSIZED(addr) \ 3154 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3155 3156 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3157 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3158 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3159 3160 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3161 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3162 kmutex_t texthole_lock; 3163 3164 char kern_bootargs[OBP_MAXPATHLEN]; 3165 char kern_bootfile[OBP_MAXPATHLEN]; 3166 3167 void 3168 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3169 { 3170 uintptr_t addr, limit; 3171 3172 addr = HEAPTEXT_BASE; 3173 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3174 3175 /* 3176 * Before we initialize the text_arena, we want to punch holes in the 3177 * underlying heaptext_arena. This guarantees that for any text 3178 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3179 */ 3180 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3181 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3182 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3183 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3184 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3185 } 3186 3187 /* 3188 * Allocate one page at the oversize to break up the text region 3189 * from the oversized region. 3190 */ 3191 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3192 (void *)limit, (void *)(limit + PAGESIZE), 3193 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3194 3195 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3196 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3197 heaptext_arena, 0, VM_SLEEP); 3198 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3199 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3200 } 3201 3202 caddr_t 3203 kobj_text_alloc(vmem_t *arena, size_t size) 3204 { 3205 caddr_t rval, better; 3206 3207 /* 3208 * First, try a sleeping allocation. 3209 */ 3210 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3211 3212 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3213 return (rval); 3214 3215 /* 3216 * We didn't get the area that we wanted. We're going to try to do an 3217 * allocation with explicit constraints. 3218 */ 3219 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3220 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3221 VM_NOSLEEP | VM_BESTFIT); 3222 3223 if (better != NULL) { 3224 /* 3225 * That worked. Free our first attempt and return. 3226 */ 3227 vmem_free(arena, rval, size); 3228 return (better); 3229 } 3230 3231 /* 3232 * That didn't work; we'll have to return our first attempt. 3233 */ 3234 return (rval); 3235 } 3236 3237 caddr_t 3238 kobj_texthole_alloc(caddr_t addr, size_t size) 3239 { 3240 int arena = HEAPTEXT_ARENA(addr); 3241 char c[30]; 3242 uintptr_t base; 3243 3244 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3245 /* 3246 * If this is an oversized allocation or it is allocated in 3247 * the nucleus data page, there is no text hole available for 3248 * it; return NULL. 3249 */ 3250 return (NULL); 3251 } 3252 3253 mutex_enter(&texthole_lock); 3254 3255 if (texthole_arena[arena] == NULL) { 3256 ASSERT(texthole_source[arena] == NULL); 3257 3258 if (arena == 0) { 3259 texthole_source[0] = vmem_create("module_text_holesrc", 3260 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3261 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3262 0, VM_SLEEP); 3263 } else { 3264 base = HEAPTEXT_BASE + 3265 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3266 3267 (void) snprintf(c, sizeof (c), 3268 "heaptext_holesrc_%d", arena); 3269 3270 texthole_source[arena] = vmem_create(c, (void *)base, 3271 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3272 0, VM_SLEEP); 3273 } 3274 3275 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3276 3277 texthole_arena[arena] = vmem_create(c, NULL, 0, 3278 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3279 texthole_source[arena], 0, VM_SLEEP); 3280 } 3281 3282 mutex_exit(&texthole_lock); 3283 3284 ASSERT(texthole_arena[arena] != NULL); 3285 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3286 return (vmem_alloc(texthole_arena[arena], size, 3287 VM_BESTFIT | VM_NOSLEEP)); 3288 } 3289 3290 void 3291 kobj_texthole_free(caddr_t addr, size_t size) 3292 { 3293 int arena = HEAPTEXT_ARENA(addr); 3294 3295 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3296 ASSERT(texthole_arena[arena] != NULL); 3297 vmem_free(texthole_arena[arena], addr, size); 3298 } 3299 3300 void 3301 release_bootstrap(void) 3302 { 3303 if (&cif_init) 3304 cif_init(); 3305 } 3306