1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/machsystm.h> 28 #include <sys/archsystm.h> 29 #include <sys/vm.h> 30 #include <sys/cpu.h> 31 #include <sys/atomic.h> 32 #include <sys/reboot.h> 33 #include <sys/kdi.h> 34 #include <sys/bootconf.h> 35 #include <sys/memlist_plat.h> 36 #include <sys/memlist_impl.h> 37 #include <sys/prom_plat.h> 38 #include <sys/prom_isa.h> 39 #include <sys/autoconf.h> 40 #include <sys/intreg.h> 41 #include <sys/ivintr.h> 42 #include <sys/fpu/fpusystm.h> 43 #include <sys/iommutsb.h> 44 #include <vm/vm_dep.h> 45 #include <vm/seg_dev.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_kpm.h> 48 #include <vm/seg_map.h> 49 #include <vm/seg_kp.h> 50 #include <sys/sysconf.h> 51 #include <vm/hat_sfmmu.h> 52 #include <sys/kobj.h> 53 #include <sys/sun4asi.h> 54 #include <sys/clconf.h> 55 #include <sys/platform_module.h> 56 #include <sys/panic.h> 57 #include <sys/cpu_sgnblk_defs.h> 58 #include <sys/clock.h> 59 #include <sys/cmn_err.h> 60 #include <sys/promif.h> 61 #include <sys/prom_debug.h> 62 #include <sys/traptrace.h> 63 #include <sys/memnode.h> 64 #include <sys/mem_cage.h> 65 #include <sys/mmu.h> 66 67 extern void setup_trap_table(void); 68 extern int cpu_intrq_setup(struct cpu *); 69 extern void cpu_intrq_register(struct cpu *); 70 extern void contig_mem_init(void); 71 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 72 extern void mach_dump_buffer_init(void); 73 extern void mach_descrip_init(void); 74 extern void mach_descrip_startup_fini(void); 75 extern void mach_memscrub(void); 76 extern void mach_fpras(void); 77 extern void mach_cpu_halt_idle(void); 78 extern void mach_hw_copy_limit(void); 79 extern void load_mach_drivers(void); 80 extern void load_tod_module(void); 81 #pragma weak load_tod_module 82 83 extern int ndata_alloc_mmfsa(struct memlist *ndata); 84 #pragma weak ndata_alloc_mmfsa 85 86 extern void cif_init(void); 87 #pragma weak cif_init 88 89 extern void parse_idprom(void); 90 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91 extern void mem_config_init(void); 92 extern void memseg_remap_init(void); 93 94 extern void mach_kpm_init(void); 95 extern void pcf_init(); 96 extern int size_pse_array(pgcnt_t, int); 97 98 /* 99 * External Data: 100 */ 101 extern int vac_size; /* cache size in bytes */ 102 extern uint_t vac_mask; /* VAC alignment consistency mask */ 103 extern uint_t vac_colors; 104 105 /* 106 * Global Data Definitions: 107 */ 108 109 /* 110 * XXX - Don't port this to new architectures 111 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 112 * 'romp' has no use with a prom with an IEEE 1275 client interface. 113 * The driver doesn't use the value, but it depends on the symbol. 114 */ 115 void *romp; /* veritas driver won't load without romp 4154976 */ 116 /* 117 * Declare these as initialized data so we can patch them. 118 */ 119 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 120 pgcnt_t segkpsize = 121 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 122 uint_t segmap_percent = 12; /* Size of segmap segment */ 123 124 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 125 int vac_copyback = 1; 126 char *cache_mode = NULL; 127 int use_mix = 1; 128 int prom_debug = 0; 129 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 151 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 152 caddr_t nalloc_base; /* beginning of nucleus allocation */ 153 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 154 caddr_t valloc_base; /* beginning of kvalloc segment */ 155 156 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 157 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 158 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 159 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 160 int kmem64_szc; /* page size code */ 161 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 162 163 uintptr_t shm_alignment; /* VAC address consistency modulus */ 164 struct memlist *phys_install; /* Total installed physical memory */ 165 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 166 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 167 struct memlist *nopp_list; /* pages with no backing page structs */ 168 struct memlist ndata; /* memlist of nucleus allocatable memory */ 169 int memexp_flag; /* memory expansion card flag */ 170 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 171 pgcnt_t obp_pages; /* Physical pages used by OBP */ 172 173 /* 174 * VM data structures 175 */ 176 long page_hashsz; /* Size of page hash table (power of two) */ 177 struct page *pp_base; /* Base of system page struct array */ 178 size_t pp_sz; /* Size in bytes of page struct array */ 179 struct page **page_hash; /* Page hash table */ 180 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 181 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 182 int pse_shift; /* log2(pse_table_size) */ 183 struct seg ktextseg; /* Segment used for kernel executable image */ 184 struct seg kvalloc; /* Segment used for "valloc" mapping */ 185 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 186 struct seg ktexthole; /* Segment used for nucleus text hole */ 187 struct seg kmapseg; /* Segment used for generic kernel mappings */ 188 struct seg kpmseg; /* Segment used for physical mapping */ 189 struct seg kdebugseg; /* Segment used for the kernel debugger */ 190 191 void *kpm_pp_base; /* Base of system kpm_page array */ 192 size_t kpm_pp_sz; /* Size of system kpm_page array */ 193 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 194 195 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 196 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 197 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 198 199 int segzio_fromheap = 0; /* zio allocations occur from heap */ 200 caddr_t segzio_base; /* Base address of segzio */ 201 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 202 203 /* 204 * debugger pages (if allocated) 205 */ 206 struct vnode kdebugvp; 207 208 /* 209 * VA range available to the debugger 210 */ 211 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 212 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 213 214 /* 215 * Segment for relocated kernel structures in 64-bit large RAM kernels 216 */ 217 struct seg kmem64; 218 219 struct memseg *memseg_free; 220 221 struct vnode unused_pages_vp; 222 223 /* 224 * VM data structures allocated early during boot. 225 */ 226 size_t pagehash_sz; 227 uint64_t memlist_sz; 228 229 char tbr_wr_addr_inited = 0; 230 231 caddr_t mpo_heap32_buf = NULL; 232 size_t mpo_heap32_bufsz = 0; 233 234 /* 235 * Static Routines: 236 */ 237 static int ndata_alloc_memseg(struct memlist *, size_t); 238 static void memlist_new(uint64_t, uint64_t, struct memlist **); 239 static void memlist_add(uint64_t, uint64_t, 240 struct memlist **, struct memlist **); 241 static void kphysm_init(void); 242 static void kvm_init(void); 243 static void install_kmem64_tte(void); 244 245 static void startup_init(void); 246 static void startup_memlist(void); 247 static void startup_modules(void); 248 static void startup_bop_gone(void); 249 static void startup_vm(void); 250 static void startup_end(void); 251 static void setup_cage_params(void); 252 static void startup_create_io_node(void); 253 254 static pgcnt_t npages; 255 static struct memlist *memlist; 256 void *memlist_end; 257 258 static pgcnt_t bop_alloc_pages; 259 static caddr_t hblk_base; 260 uint_t hblk_alloc_dynamic = 0; 261 uint_t hblk1_min = H1MIN; 262 263 264 /* 265 * Hooks for unsupported platforms and down-rev firmware 266 */ 267 int iam_positron(void); 268 #pragma weak iam_positron 269 static void do_prom_version_check(void); 270 271 /* 272 * After receiving a thermal interrupt, this is the number of seconds 273 * to delay before shutting off the system, assuming 274 * shutdown fails. Use /etc/system to change the delay if this isn't 275 * large enough. 276 */ 277 int thermal_powerdown_delay = 1200; 278 279 /* 280 * Used to hold off page relocations into the cage until OBP has completed 281 * its boot-time handoff of its resources to the kernel. 282 */ 283 int page_relocate_ready = 0; 284 285 /* 286 * Indicate if kmem64 allocation was done in small chunks 287 */ 288 int kmem64_smchunks = 0; 289 290 /* 291 * Enable some debugging messages concerning memory usage... 292 */ 293 #ifdef DEBUGGING_MEM 294 static int debugging_mem; 295 static void 296 printmemlist(char *title, struct memlist *list) 297 { 298 if (!debugging_mem) 299 return; 300 301 printf("%s\n", title); 302 303 while (list) { 304 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 305 (uint32_t)(list->address >> 32), (uint32_t)list->address, 306 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 307 list = list->next; 308 } 309 } 310 311 void 312 printmemseg(struct memseg *memseg) 313 { 314 if (!debugging_mem) 315 return; 316 317 printf("memseg\n"); 318 319 while (memseg) { 320 prom_printf("\tpage = 0x%p, epage = 0x%p, " 321 "pfn = 0x%x, epfn = 0x%x\n", 322 memseg->pages, memseg->epages, 323 memseg->pages_base, memseg->pages_end); 324 memseg = memseg->next; 325 } 326 } 327 328 #define debug_pause(str) halt((str)) 329 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 330 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 331 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 332 #define MPRINTF3(str, a, b, c) \ 333 if (debugging_mem) prom_printf((str), (a), (b), (c)) 334 #else /* DEBUGGING_MEM */ 335 #define MPRINTF(str) 336 #define MPRINTF1(str, a) 337 #define MPRINTF2(str, a, b) 338 #define MPRINTF3(str, a, b, c) 339 #endif /* DEBUGGING_MEM */ 340 341 342 /* 343 * 344 * Kernel's Virtual Memory Layout. 345 * /-----------------------\ 346 * 0xFFFFFFFF.FFFFFFFF -| |- 347 * | OBP's virtual page | 348 * | tables | 349 * 0xFFFFFFFC.00000000 -|-----------------------|- 350 * : : 351 * : : 352 * -|-----------------------|- 353 * | segzio | (base and size vary) 354 * 0xFFFFFE00.00000000 -|-----------------------|- 355 * | | Ultrasparc I/II support 356 * | segkpm segment | up to 2TB of physical 357 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 358 * | | 359 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 360 * : : 361 * : : 362 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 363 * | | ^ 364 * | UltraSPARC I/II call | | 365 * | bug requires an extra | | 366 * | 4 GB of space between | | 367 * | hole and used RAM | | 368 * | | | 369 * 0xFFFFF800.00000000 -|-----------------------|- | 370 * | | | 371 * | Virtual Address Hole | UltraSPARC 372 * | on UltraSPARC I/II | I/II * ONLY * 373 * | | | 374 * 0x00000800.00000000 -|-----------------------|- | 375 * | | | 376 * | UltraSPARC I/II call | | 377 * | bug requires an extra | | 378 * | 4 GB of space between | | 379 * | hole and used RAM | | 380 * | | v 381 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 382 * : : ^ 383 * : : | 384 * |-----------------------| | 385 * | | | 386 * | ecache flush area | | 387 * | (twice largest e$) | | 388 * | | | 389 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 390 * | overmapped area | alignend_end | 391 * | (kmem64_alignsize | | 392 * | boundary) | | 393 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 394 * | | | 395 * | 64-bit kernel ONLY | | 396 * | | | 397 * | kmem64 segment | | 398 * | | | 399 * | (Relocated extra HME | Approximately 400 * | block allocations, | 1 TB of virtual 401 * | memnode freelists, | address space 402 * | HME hash buckets, | | 403 * | mml_table, kpmp_table,| | 404 * | page_t array and | | 405 * | hashblock pool to | | 406 * | avoid hard-coded | | 407 * | 32-bit vaddr | | 408 * | limitations) | | 409 * | | v 410 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 411 * | | 412 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 413 * | | 414 * 0x00000300.00000000 -|-----------------------|- SYSBASE 415 * : : 416 * : : 417 * -|-----------------------|- 418 * | | 419 * | segmap segment | SEGMAPSIZE (1/8th physmem, 420 * | | 256G MAX) 421 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 422 * : : 423 * : : 424 * -|-----------------------|- 425 * | | 426 * | segkp | SEGKPSIZE (2GB) 427 * | | 428 * | | 429 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 430 * | | 431 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 432 * | | (SEGKPBASE - 0x400000) 433 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 434 * | | (MEMSCRUBBASE - NCARGS) 435 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 436 * | | (ARGSBASE - PPMAPSIZE) 437 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 438 * | | 439 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 440 * | | 441 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 442 * : : 443 * : : 444 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 445 * | | 446 * | OBP | 447 * | | 448 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 449 * | kmdb | 450 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 451 * : : 452 * : : 453 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 454 * | | 455 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 456 * | | ~64MB) 457 * 0x00000000.70002000 -|-----------------------| 458 * | panicbuf | 459 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 460 * | boot-time | 461 * | temporary space | 462 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 463 * : : 464 * : : 465 * | | 466 * |-----------------------|- econtig32 467 * | vm structures | 468 * 0x00000000.01C00000 |-----------------------|- nalloc_end 469 * | TSBs | 470 * |-----------------------|- end/nalloc_base 471 * | kernel data & bss | 472 * 0x00000000.01800000 -|-----------------------| 473 * : nucleus text hole : 474 * 0x00000000.01400000 -|-----------------------| 475 * : : 476 * |-----------------------| 477 * | module text | 478 * |-----------------------|- e_text/modtext 479 * | kernel text | 480 * |-----------------------| 481 * | trap table (48k) | 482 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 483 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 484 * |-----------------------| 485 * | | 486 * | invalid | 487 * | | 488 * 0x00000000.00000000 _|_______________________| 489 * 490 * 491 * 492 * 32-bit User Virtual Memory Layout. 493 * /-----------------------\ 494 * | | 495 * | invalid | 496 * | | 497 * 0xFFC00000 -|-----------------------|- USERLIMIT 498 * | user stack | 499 * : : 500 * : : 501 * : : 502 * | user data | 503 * -|-----------------------|- 504 * | user text | 505 * 0x00002000 -|-----------------------|- 506 * | invalid | 507 * 0x00000000 _|_______________________| 508 * 509 * 510 * 511 * 64-bit User Virtual Memory Layout. 512 * /-----------------------\ 513 * | | 514 * | invalid | 515 * | | 516 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 517 * | user stack | 518 * : : 519 * : : 520 * : : 521 * | user data | 522 * -|-----------------------|- 523 * | user text | 524 * 0x00000000.01000000 -|-----------------------|- 525 * | invalid | 526 * 0x00000000.00000000 _|_______________________| 527 */ 528 529 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 530 extern uint64_t ecache_flush_address(void); 531 532 #pragma weak load_platform_modules 533 #pragma weak plat_startup_memlist 534 #pragma weak ecache_init_scrub_flush_area 535 #pragma weak ecache_flush_address 536 537 538 /* 539 * By default the DR Cage is enabled for maximum OS 540 * MPSS performance. Users needing to disable the cage mechanism 541 * can set this variable to zero via /etc/system. 542 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 543 * will result in loss of DR functionality. 544 * Platforms wishing to disable kernel Cage by default 545 * should do so in their set_platform_defaults() routine. 546 */ 547 int kernel_cage_enable = 1; 548 549 static void 550 setup_cage_params(void) 551 { 552 void (*func)(void); 553 554 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 555 if (func != NULL) { 556 (*func)(); 557 return; 558 } 559 560 if (kernel_cage_enable == 0) { 561 return; 562 } 563 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 564 565 if (kcage_on) { 566 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 567 } else { 568 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 569 } 570 571 } 572 573 /* 574 * Machine-dependent startup code 575 */ 576 void 577 startup(void) 578 { 579 startup_init(); 580 if (&startup_platform) 581 startup_platform(); 582 startup_memlist(); 583 startup_modules(); 584 setup_cage_params(); 585 startup_bop_gone(); 586 startup_vm(); 587 startup_end(); 588 } 589 590 struct regs sync_reg_buf; 591 uint64_t sync_tt; 592 593 void 594 sync_handler(void) 595 { 596 struct panic_trap_info ti; 597 int i; 598 599 /* 600 * Prevent trying to talk to the other CPUs since they are 601 * sitting in the prom and won't reply. 602 */ 603 for (i = 0; i < NCPU; i++) { 604 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 605 cpu[i]->cpu_flags &= ~CPU_READY; 606 cpu[i]->cpu_flags |= CPU_QUIESCED; 607 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 608 } 609 } 610 611 /* 612 * We've managed to get here without going through the 613 * normal panic code path. Try and save some useful 614 * information. 615 */ 616 if (!panicstr && (curthread->t_panic_trap == NULL)) { 617 ti.trap_type = sync_tt; 618 ti.trap_regs = &sync_reg_buf; 619 ti.trap_addr = NULL; 620 ti.trap_mmu_fsr = 0x0; 621 622 curthread->t_panic_trap = &ti; 623 } 624 625 /* 626 * If we're re-entering the panic path, update the signature 627 * block so that the SC knows we're in the second part of panic. 628 */ 629 if (panicstr) 630 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 631 632 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 633 panic("sync initiated"); 634 } 635 636 637 static void 638 startup_init(void) 639 { 640 /* 641 * We want to save the registers while we're still in OBP 642 * so that we know they haven't been fiddled with since. 643 * (In principle, OBP can't change them just because it 644 * makes a callback, but we'd rather not depend on that 645 * behavior.) 646 */ 647 char sync_str[] = 648 "warning @ warning off : sync " 649 "%%tl-c %%tstate h# %p x! " 650 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 651 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 652 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 653 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 654 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 655 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 656 "%%y h# %p l! %%tl-c %%tt h# %p x! " 657 "sync ; warning !"; 658 659 /* 660 * 20 == num of %p substrings 661 * 16 == max num of chars %p will expand to. 662 */ 663 char bp[sizeof (sync_str) + 16 * 20]; 664 665 /* 666 * Initialize ptl1 stack for the 1st CPU. 667 */ 668 ptl1_init_cpu(&cpu0); 669 670 /* 671 * Initialize the address map for cache consistent mappings 672 * to random pages; must be done after vac_size is set. 673 */ 674 ppmapinit(); 675 676 /* 677 * Initialize the PROM callback handler. 678 */ 679 init_vx_handler(); 680 681 /* 682 * have prom call sync_callback() to handle the sync and 683 * save some useful information which will be stored in the 684 * core file later. 685 */ 686 (void) sprintf((char *)bp, sync_str, 687 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 688 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 689 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 690 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 691 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 692 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 693 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 694 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 695 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 696 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 697 prom_interpret(bp, 0, 0, 0, 0, 0); 698 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 699 } 700 701 702 size_t 703 calc_pp_sz(pgcnt_t npages) 704 { 705 706 return (npages * sizeof (struct page)); 707 } 708 709 size_t 710 calc_kpmpp_sz(pgcnt_t npages) 711 { 712 713 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 714 kpm_pgsz = 1ull << kpm_pgshft; 715 kpm_pgoff = kpm_pgsz - 1; 716 kpmp2pshft = kpm_pgshft - PAGESHIFT; 717 kpmpnpgs = 1 << kpmp2pshft; 718 719 if (kpm_smallpages == 0) { 720 /* 721 * Avoid fragmentation problems in kphysm_init() 722 * by allocating for all of physical memory 723 */ 724 kpm_npages = ptokpmpr(physinstalled); 725 return (kpm_npages * sizeof (kpm_page_t)); 726 } else { 727 kpm_npages = npages; 728 return (kpm_npages * sizeof (kpm_spage_t)); 729 } 730 } 731 732 size_t 733 calc_pagehash_sz(pgcnt_t npages) 734 { 735 736 /* 737 * The page structure hash table size is a power of 2 738 * such that the average hash chain length is PAGE_HASHAVELEN. 739 */ 740 page_hashsz = npages / PAGE_HASHAVELEN; 741 page_hashsz = 1 << highbit(page_hashsz); 742 return (page_hashsz * sizeof (struct page *)); 743 } 744 745 int testkmem64_smchunks = 0; 746 747 int 748 alloc_kmem64(caddr_t base, caddr_t end) 749 { 750 int i; 751 caddr_t aligned_end = NULL; 752 753 if (testkmem64_smchunks) 754 return (1); 755 756 /* 757 * Make one large memory alloc after figuring out the 64-bit size. This 758 * will enable use of the largest page size appropriate for the system 759 * architecture. 760 */ 761 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 762 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 763 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 764 size_t alloc_size, alignsize; 765 #if !defined(C_OBP) 766 unsigned long long pa; 767 #endif /* !C_OBP */ 768 769 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 770 continue; 771 alignsize = TTEBYTES(i); 772 kmem64_szc = i; 773 774 /* limit page size for small memory */ 775 if (mmu_btop(alignsize) > (npages >> 2)) 776 continue; 777 778 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 779 alloc_size = aligned_end - base; 780 #if !defined(C_OBP) 781 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 782 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 783 kmem64_pabase = pa; 784 kmem64_aligned_end = aligned_end; 785 install_kmem64_tte(); 786 break; 787 } else { 788 prom_free_phys(alloc_size, pa); 789 } 790 } 791 #else /* !C_OBP */ 792 if (prom_alloc(base, alloc_size, alignsize) == base) { 793 kmem64_pabase = va_to_pa(kmem64_base); 794 kmem64_aligned_end = aligned_end; 795 break; 796 } 797 #endif /* !C_OBP */ 798 if (i == TTE8K) { 799 #ifdef sun4v 800 /* return failure to try small allocations */ 801 return (1); 802 #else 803 prom_panic("kmem64 allocation failure"); 804 #endif 805 } 806 } 807 ASSERT(aligned_end != NULL); 808 return (0); 809 } 810 811 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 812 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 813 814 #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 815 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 816 (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 817 818 #if !defined(C_OBP) 819 /* 820 * Install a temporary tte handler in OBP for kmem64 area. 821 * 822 * We map kmem64 area with large pages before the trap table is taken 823 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 824 * the same area. Duplicate tlb entries with different page sizes 825 * cause unpredicatble behavior. To avoid this, we don't create 826 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 827 * OBP). Instead, we manage translations with a temporary va>tte-data 828 * handler (kmem64-tte). This handler is replaced by unix-tte when 829 * the trap table is taken over. 830 * 831 * The temporary handler knows the physical address of the kmem64 832 * area. It uses the prom's pgmap@ Forth word for other addresses. 833 * 834 * We have to use BOP_ALLOC() method for C-OBP platforms because 835 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 836 * sun4u platforms. On sun4u we flush tlb after trap table is taken 837 * over if we use large pages for kernel heap and kmem64. Since sun4u 838 * prom (unlike sun4v) calls va>tte-data first for client address 839 * translation prom's ttes for kmem64 can't get into TLB even if we 840 * later switch to prom's trap table again. C-OBP uses 4M pages for 841 * client mappings when possible so on all platforms we get the 842 * benefit from large mappings for kmem64 area immediately during 843 * boot. 844 * 845 * pseudo code: 846 * if (context != 0) { 847 * return false 848 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 849 * tte = tte_template + 850 * (((miss_va & pagemask) - kmem64_base)); 851 * return tte, true 852 * } else { 853 * return pgmap@ result 854 * } 855 */ 856 char kmem64_obp_str[] = 857 "h# %lx constant kmem64-base " 858 "h# %lx constant kmem64-end " 859 "h# %lx constant kmem64-pagemask " 860 "h# %lx constant kmem64-template " 861 862 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 863 " if ( addr ) " 864 " drop false exit then ( false ) " 865 " dup kmem64-base kmem64-end within if ( addr ) " 866 " kmem64-pagemask and ( addr' ) " 867 " kmem64-base - ( addr' ) " 868 " kmem64-template + ( tte ) " 869 " true ( tte true ) " 870 " else ( addr ) " 871 " pgmap@ ( tte ) " 872 " dup 0< if true else drop false then ( tte true | false ) " 873 " then ( tte true | false ) " 874 "; " 875 876 "' kmem64-tte is va>tte-data " 877 ; 878 879 static void 880 install_kmem64_tte() 881 { 882 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 883 tte_t tte; 884 885 PRM_DEBUG(kmem64_pabase); 886 PRM_DEBUG(kmem64_szc); 887 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 888 PROC_DATA | HAT_NOSYNC | HAT_ATTR_NOSOFTEXEC, kmem64_szc); 889 PRM_DEBUG(tte.ll); 890 (void) sprintf(b, kmem64_obp_str, 891 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 892 ASSERT(strlen(b) < sizeof (b)); 893 prom_interpret(b, 0, 0, 0, 0, 0); 894 } 895 #endif /* !C_OBP */ 896 897 /* 898 * As OBP takes up some RAM when the system boots, pages will already be "lost" 899 * to the system and reflected in npages by the time we see it. 900 * 901 * We only want to allocate kernel structures in the 64-bit virtual address 902 * space on systems with enough RAM to make the overhead of keeping track of 903 * an extra kernel memory segment worthwhile. 904 * 905 * Since OBP has already performed its memory allocations by this point, if we 906 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 907 * memory in the 64-bit virtual address space; otherwise keep allocations 908 * contiguous with we've mapped so far in the 32-bit virtual address space. 909 */ 910 #define MINMOVE_RAM_MB ((size_t)1900) 911 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 912 #define BYTES_TO_MB(b) ((b) / 1048576ul) 913 914 pgcnt_t tune_npages = (pgcnt_t) 915 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 916 917 #pragma weak page_set_colorequiv_arr_cpu 918 extern void page_set_colorequiv_arr_cpu(void); 919 extern void page_set_colorequiv_arr(void); 920 921 static pgcnt_t ramdisk_npages; 922 static struct memlist *old_phys_avail; 923 924 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 925 926 static void 927 startup_memlist(void) 928 { 929 size_t hmehash_sz, pagelist_sz, tt_sz; 930 size_t psetable_sz; 931 caddr_t alloc_base; 932 caddr_t memspace; 933 struct memlist *cur; 934 size_t syslimit = (size_t)SYSLIMIT; 935 size_t sysbase = (size_t)SYSBASE; 936 937 /* 938 * Initialize enough of the system to allow kmem_alloc to work by 939 * calling boot to allocate its memory until the time that 940 * kvm_init is completed. The page structs are allocated after 941 * rounding up end to the nearest page boundary; the memsegs are 942 * initialized and the space they use comes from the kernel heap. 943 * With appropriate initialization, they can be reallocated later 944 * to a size appropriate for the machine's configuration. 945 * 946 * At this point, memory is allocated for things that will never 947 * need to be freed, this used to be "valloced". This allows a 948 * savings as the pages don't need page structures to describe 949 * them because them will not be managed by the vm system. 950 */ 951 952 /* 953 * We're loaded by boot with the following configuration (as 954 * specified in the sun4u/conf/Mapfile): 955 * 956 * text: 4 MB chunk aligned on a 4MB boundary 957 * data & bss: 4 MB chunk aligned on a 4MB boundary 958 * 959 * These two chunks will eventually be mapped by 2 locked 4MB 960 * ttes and will represent the nucleus of the kernel. This gives 961 * us some free space that is already allocated, some or all of 962 * which is made available to kernel module text. 963 * 964 * The free space in the data-bss chunk is used for nucleus 965 * allocatable data structures and we reserve it using the 966 * nalloc_base and nalloc_end variables. This space is currently 967 * being used for hat data structures required for tlb miss 968 * handling operations. We align nalloc_base to a l2 cache 969 * linesize because this is the line size the hardware uses to 970 * maintain cache coherency. 971 * 512K is carved out for module data. 972 */ 973 974 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 975 e_moddata = moddata + MODDATA; 976 nalloc_base = e_moddata; 977 978 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 979 valloc_base = nalloc_base; 980 981 /* 982 * Calculate the start of the data segment. 983 */ 984 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 985 prom_panic("nucleus data overflow"); 986 987 PRM_DEBUG(moddata); 988 PRM_DEBUG(nalloc_base); 989 PRM_DEBUG(nalloc_end); 990 991 /* 992 * Remember any slop after e_text so we can give it to the modules. 993 */ 994 PRM_DEBUG(e_text); 995 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 996 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 997 prom_panic("nucleus text overflow"); 998 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 999 modtext; 1000 PRM_DEBUG(modtext); 1001 PRM_DEBUG(modtext_sz); 1002 1003 init_boot_memlists(); 1004 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1005 &boot_physavail, &boot_physavail_len, 1006 &boot_virtavail, &boot_virtavail_len); 1007 1008 /* 1009 * Remember what the physically available highest page is 1010 * so that dumpsys works properly, and find out how much 1011 * memory is installed. 1012 */ 1013 installed_top_size_memlist_array(boot_physinstalled, 1014 boot_physinstalled_len, &physmax, &physinstalled); 1015 PRM_DEBUG(physinstalled); 1016 PRM_DEBUG(physmax); 1017 1018 /* Fill out memory nodes config structure */ 1019 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1020 1021 /* 1022 * npages is the maximum of available physical memory possible. 1023 * (ie. it will never be more than this) 1024 * 1025 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1026 * using phys_avail will underestimate what will end up being freed. 1027 * A better initial guess is just total memory minus the kernel text 1028 */ 1029 npages = physinstalled - btop(MMU_PAGESIZE4M); 1030 1031 /* 1032 * First allocate things that can go in the nucleus data page 1033 * (fault status, TSBs, dmv, CPUs) 1034 */ 1035 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1036 1037 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1038 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1039 1040 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1041 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1042 1043 if (ndata_alloc_dmv(&ndata) != 0) 1044 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1045 1046 if (ndata_alloc_page_mutexs(&ndata) != 0) 1047 cmn_err(CE_PANIC, 1048 "no more nucleus memory after page free lists alloc"); 1049 1050 if (ndata_alloc_hat(&ndata, npages) != 0) 1051 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1052 1053 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1054 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1055 1056 /* 1057 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1058 * 1059 * There are comments all over the SFMMU code warning of dire 1060 * consequences if the TSBs are moved out of 32-bit space. This 1061 * is largely because the asm code uses "sethi %hi(addr)"-type 1062 * instructions which will not provide the expected result if the 1063 * address is a 64-bit one. 1064 * 1065 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1066 */ 1067 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1068 PRM_DEBUG(alloc_base); 1069 1070 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1071 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1072 PRM_DEBUG(alloc_base); 1073 1074 /* 1075 * Allocate IOMMU TSB array. We do this here so that the physical 1076 * memory gets deducted from the PROM's physical memory list. 1077 */ 1078 alloc_base = iommu_tsb_init(alloc_base); 1079 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1080 PRM_DEBUG(alloc_base); 1081 1082 /* 1083 * Allow for an early allocation of physically contiguous memory. 1084 */ 1085 alloc_base = contig_mem_prealloc(alloc_base, npages); 1086 1087 /* 1088 * Platforms like Starcat and OPL need special structures assigned in 1089 * 32-bit virtual address space because their probing routines execute 1090 * FCode, and FCode can't handle 64-bit virtual addresses... 1091 */ 1092 if (&plat_startup_memlist) { 1093 alloc_base = plat_startup_memlist(alloc_base); 1094 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1095 ecache_alignsize); 1096 PRM_DEBUG(alloc_base); 1097 } 1098 1099 /* 1100 * Save off where the contiguous allocations to date have ended 1101 * in econtig32. 1102 */ 1103 econtig32 = alloc_base; 1104 PRM_DEBUG(econtig32); 1105 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1106 cmn_err(CE_PANIC, "econtig32 too big"); 1107 1108 pp_sz = calc_pp_sz(npages); 1109 PRM_DEBUG(pp_sz); 1110 if (kpm_enable) { 1111 kpm_pp_sz = calc_kpmpp_sz(npages); 1112 PRM_DEBUG(kpm_pp_sz); 1113 } 1114 1115 hmehash_sz = calc_hmehash_sz(npages); 1116 PRM_DEBUG(hmehash_sz); 1117 1118 pagehash_sz = calc_pagehash_sz(npages); 1119 PRM_DEBUG(pagehash_sz); 1120 1121 pagelist_sz = calc_free_pagelist_sz(); 1122 PRM_DEBUG(pagelist_sz); 1123 1124 #ifdef TRAPTRACE 1125 tt_sz = calc_traptrace_sz(); 1126 PRM_DEBUG(tt_sz); 1127 #else 1128 tt_sz = 0; 1129 #endif /* TRAPTRACE */ 1130 1131 /* 1132 * Place the array that protects pp->p_selock in the kmem64 wad. 1133 */ 1134 pse_shift = size_pse_array(npages, max_ncpus); 1135 PRM_DEBUG(pse_shift); 1136 pse_table_size = 1 << pse_shift; 1137 PRM_DEBUG(pse_table_size); 1138 psetable_sz = roundup( 1139 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1140 PRM_DEBUG(psetable_sz); 1141 1142 /* 1143 * Now allocate the whole wad 1144 */ 1145 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1146 pagelist_sz + tt_sz + psetable_sz; 1147 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1148 kmem64_base = (caddr_t)syslimit; 1149 kmem64_end = kmem64_base + kmem64_sz; 1150 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1151 /* 1152 * Attempt for kmem64 to allocate one big 1153 * contiguous chunk of memory failed. 1154 * We get here because we are sun4v. 1155 * We will proceed by breaking up 1156 * the allocation into two attempts. 1157 * First, we allocate kpm_pp_sz, hmehash_sz, 1158 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1159 * one contiguous chunk. This is a much smaller 1160 * chunk and we should get it, if not we panic. 1161 * Note that hmehash and tt need to be physically 1162 * (in the real address sense) contiguous. 1163 * Next, we use bop_alloc_chunk() to 1164 * to allocate the page_t structures. 1165 * This will allow the page_t to be allocated 1166 * in multiple smaller chunks. 1167 * In doing so, the assumption that page_t is 1168 * physically contiguous no longer hold, this is ok 1169 * for sun4v but not for sun4u. 1170 */ 1171 size_t tmp_size; 1172 caddr_t tmp_base; 1173 1174 pp_sz = roundup(pp_sz, PAGESIZE); 1175 1176 /* 1177 * Allocate kpm_pp_sz, hmehash_sz, 1178 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1179 */ 1180 tmp_base = kmem64_base + pp_sz; 1181 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1182 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1183 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1184 prom_panic("kmem64 prom_alloc contig failed"); 1185 PRM_DEBUG(tmp_base); 1186 PRM_DEBUG(tmp_size); 1187 1188 /* 1189 * Allocate the page_ts 1190 */ 1191 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1192 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1193 PRM_DEBUG(kmem64_base); 1194 PRM_DEBUG(pp_sz); 1195 1196 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1197 ASSERT(kmem64_aligned_end >= kmem64_end); 1198 1199 kmem64_smchunks = 1; 1200 } else { 1201 1202 /* 1203 * We need to adjust pp_sz for the normal 1204 * case where kmem64 can allocate one large chunk 1205 */ 1206 if (kpm_smallpages == 0) { 1207 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1208 } else { 1209 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1210 sizeof (kpm_spage_t)); 1211 } 1212 pp_sz = npages * sizeof (struct page); 1213 } 1214 1215 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1216 cmn_err(CE_PANIC, "not enough kmem64 space"); 1217 PRM_DEBUG(kmem64_base); 1218 PRM_DEBUG(kmem64_end); 1219 PRM_DEBUG(kmem64_aligned_end); 1220 1221 /* 1222 * ... and divy it up 1223 */ 1224 alloc_base = kmem64_base; 1225 1226 pp_base = (page_t *)alloc_base; 1227 alloc_base += pp_sz; 1228 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1229 PRM_DEBUG(pp_base); 1230 PRM_DEBUG(npages); 1231 1232 if (kpm_enable) { 1233 kpm_pp_base = alloc_base; 1234 if (kpm_smallpages == 0) { 1235 /* kpm_npages based on physinstalled, don't reset */ 1236 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1237 } else { 1238 kpm_npages = ptokpmpr(npages); 1239 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1240 } 1241 alloc_base += kpm_pp_sz; 1242 alloc_base = 1243 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1244 PRM_DEBUG(kpm_pp_base); 1245 } 1246 1247 alloc_base = alloc_hmehash(alloc_base); 1248 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1249 PRM_DEBUG(alloc_base); 1250 1251 page_hash = (page_t **)alloc_base; 1252 alloc_base += pagehash_sz; 1253 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1254 PRM_DEBUG(page_hash); 1255 1256 alloc_base = alloc_page_freelists(alloc_base); 1257 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1258 PRM_DEBUG(alloc_base); 1259 1260 #ifdef TRAPTRACE 1261 ttrace_buf = alloc_base; 1262 alloc_base += tt_sz; 1263 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1264 PRM_DEBUG(alloc_base); 1265 #endif /* TRAPTRACE */ 1266 1267 pse_mutex = (pad_mutex_t *)alloc_base; 1268 alloc_base += psetable_sz; 1269 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1270 PRM_DEBUG(alloc_base); 1271 1272 /* 1273 * Note that if we use small chunk allocations for 1274 * kmem64, we need to ensure kmem64_end is the same as 1275 * kmem64_aligned_end to prevent subsequent logic from 1276 * trying to reuse the overmapping. 1277 * Otherwise we adjust kmem64_end to what we really allocated. 1278 */ 1279 if (kmem64_smchunks) { 1280 kmem64_end = kmem64_aligned_end; 1281 } else { 1282 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1283 } 1284 kmem64_sz = kmem64_end - kmem64_base; 1285 1286 if (&ecache_init_scrub_flush_area) { 1287 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1288 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1289 } 1290 1291 /* 1292 * If physmem is patched to be non-zero, use it instead of 1293 * the monitor value unless physmem is larger than the total 1294 * amount of memory on hand. 1295 */ 1296 if (physmem == 0 || physmem > npages) 1297 physmem = npages; 1298 1299 /* 1300 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1301 * is mounted as root. This memory is held down by OBP and unlike 1302 * the stub boot_archive is never released. 1303 * 1304 * In order to get things sized correctly on lower memory 1305 * machines (where the memory used by the ramdisk represents 1306 * a significant portion of memory), physmem is adjusted. 1307 * 1308 * This is done by subtracting the ramdisk_size which is set 1309 * to the size of the ramdisk (in Kb) in /etc/system at the 1310 * time the miniroot archive is constructed. 1311 */ 1312 if (root_is_ramdisk == B_TRUE) { 1313 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1314 physmem -= ramdisk_npages; 1315 } 1316 1317 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1318 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1319 1320 /* 1321 * Allocate space for the interrupt vector table. 1322 */ 1323 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1324 if (memspace != (caddr_t)intr_vec_table) 1325 prom_panic("interrupt vector table allocation failure"); 1326 1327 /* 1328 * Between now and when we finish copying in the memory lists, 1329 * allocations happen so the space gets fragmented and the 1330 * lists longer. Leave enough space for lists twice as 1331 * long as we have now; then roundup to a pagesize. 1332 */ 1333 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1334 prom_phys_avail_len() + prom_virt_avail_len()); 1335 memlist_sz *= 2; 1336 memlist_sz = roundup(memlist_sz, PAGESIZE); 1337 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1338 if (memspace == NULL) 1339 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1340 1341 memlist = (struct memlist *)memspace; 1342 memlist_end = (char *)memspace + memlist_sz; 1343 PRM_DEBUG(memlist); 1344 PRM_DEBUG(memlist_end); 1345 1346 PRM_DEBUG(sysbase); 1347 PRM_DEBUG(syslimit); 1348 kernelheap_init((void *)sysbase, (void *)syslimit, 1349 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1350 1351 /* 1352 * Take the most current snapshot we can by calling mem-update. 1353 */ 1354 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1355 &boot_physavail, &boot_physavail_len, 1356 &boot_virtavail, &boot_virtavail_len); 1357 1358 /* 1359 * Remove the space used by prom_alloc from the kernel heap 1360 * plus the area actually used by the OBP (if any) 1361 * ignoring virtual addresses in virt_avail, above syslimit. 1362 */ 1363 virt_avail = memlist; 1364 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1365 1366 for (cur = virt_avail; cur->next; cur = cur->next) { 1367 uint64_t range_base, range_size; 1368 1369 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1370 continue; 1371 if (range_base >= (uint64_t)syslimit) 1372 break; 1373 /* 1374 * Limit the range to end at syslimit. 1375 */ 1376 range_size = MIN(cur->next->address, 1377 (uint64_t)syslimit) - range_base; 1378 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1379 0, 0, (void *)range_base, (void *)(range_base + range_size), 1380 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1381 } 1382 1383 phys_avail = memlist; 1384 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1385 1386 /* 1387 * Add any extra memory at the end of the ndata region if there's at 1388 * least a page to add. There might be a few more pages available in 1389 * the middle of the ndata region, but for now they are ignored. 1390 */ 1391 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1392 if (nalloc_base == NULL) 1393 nalloc_base = nalloc_end; 1394 ndata_remain_sz = nalloc_end - nalloc_base; 1395 1396 /* 1397 * Copy physinstalled list into kernel space. 1398 */ 1399 phys_install = memlist; 1400 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1401 1402 /* 1403 * Create list of physical addrs we don't need pp's for: 1404 * kernel text 4M page 1405 * kernel data 4M page - ndata_remain_sz 1406 * kmem64 pages 1407 * 1408 * NB if adding any pages here, make sure no kpm page 1409 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1410 */ 1411 nopp_list = memlist; 1412 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1413 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1414 &memlist, &nopp_list); 1415 1416 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1417 if (!kmem64_smchunks) 1418 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1419 1420 if ((caddr_t)memlist > (memspace + memlist_sz)) 1421 prom_panic("memlist overflow"); 1422 1423 /* 1424 * Size the pcf array based on the number of cpus in the box at 1425 * boot time. 1426 */ 1427 pcf_init(); 1428 1429 /* 1430 * Initialize the page structures from the memory lists. 1431 */ 1432 kphysm_init(); 1433 1434 availrmem_initial = availrmem = freemem; 1435 PRM_DEBUG(availrmem); 1436 1437 /* 1438 * Some of the locks depend on page_hashsz being set! 1439 * kmem_init() depends on this; so, keep it here. 1440 */ 1441 page_lock_init(); 1442 1443 /* 1444 * Initialize kernel memory allocator. 1445 */ 1446 kmem_init(); 1447 1448 /* 1449 * Factor in colorequiv to check additional 'equivalent' bins 1450 */ 1451 if (&page_set_colorequiv_arr_cpu != NULL) 1452 page_set_colorequiv_arr_cpu(); 1453 else 1454 page_set_colorequiv_arr(); 1455 1456 /* 1457 * Initialize bp_mapin(). 1458 */ 1459 bp_init(shm_alignment, HAT_STRICTORDER); 1460 1461 /* 1462 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1463 * vector data structures and MPO mblock structs from the 32-bit heap. 1464 */ 1465 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1466 panicbuf, panicbuf + PANICBUFSIZE, 1467 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1468 1469 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1470 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1471 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1472 1473 if (mpo_heap32_bufsz > (size_t)0) { 1474 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1475 PAGESIZE, 0, 0, mpo_heap32_buf, 1476 mpo_heap32_buf + mpo_heap32_bufsz, 1477 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1478 } 1479 mem_config_init(); 1480 } 1481 1482 static void 1483 startup_modules(void) 1484 { 1485 int nhblk1, nhblk8; 1486 size_t nhblksz; 1487 pgcnt_t pages_per_hblk; 1488 size_t hme8blk_sz, hme1blk_sz; 1489 1490 /* 1491 * Let the platforms have a chance to change default 1492 * values before reading system file. 1493 */ 1494 if (&set_platform_defaults) 1495 set_platform_defaults(); 1496 1497 /* 1498 * Calculate default settings of system parameters based upon 1499 * maxusers, yet allow to be overridden via the /etc/system file. 1500 */ 1501 param_calc(0); 1502 1503 mod_setup(); 1504 1505 /* 1506 * If this is a positron, complain and halt. 1507 */ 1508 if (&iam_positron && iam_positron()) { 1509 cmn_err(CE_WARN, "This hardware platform is not supported" 1510 " by this release of Solaris.\n"); 1511 #ifdef DEBUG 1512 prom_enter_mon(); /* Type 'go' to resume */ 1513 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1514 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1515 1516 #else /* DEBUG */ 1517 halt(0); 1518 #endif /* DEBUG */ 1519 } 1520 1521 /* 1522 * If we are running firmware that isn't 64-bit ready 1523 * then complain and halt. 1524 */ 1525 do_prom_version_check(); 1526 1527 /* 1528 * Initialize system parameters 1529 */ 1530 param_init(); 1531 1532 /* 1533 * maxmem is the amount of physical memory we're playing with. 1534 */ 1535 maxmem = physmem; 1536 1537 /* Set segkp limits. */ 1538 ncbase = kdi_segdebugbase; 1539 ncend = kdi_segdebugbase; 1540 1541 /* 1542 * Initialize the hat layer. 1543 */ 1544 hat_init(); 1545 1546 /* 1547 * Initialize segment management stuff. 1548 */ 1549 seg_init(); 1550 1551 /* 1552 * Create the va>tte handler, so the prom can understand 1553 * kernel translations. The handler is installed later, just 1554 * as we are about to take over the trap table from the prom. 1555 */ 1556 create_va_to_tte(); 1557 1558 /* 1559 * Load the forthdebugger (optional) 1560 */ 1561 forthdebug_init(); 1562 1563 /* 1564 * Create OBP node for console input callbacks 1565 * if it is needed. 1566 */ 1567 startup_create_io_node(); 1568 1569 if (modloadonly("fs", "specfs") == -1) 1570 halt("Can't load specfs"); 1571 1572 if (modloadonly("fs", "devfs") == -1) 1573 halt("Can't load devfs"); 1574 1575 if (modloadonly("misc", "swapgeneric") == -1) 1576 halt("Can't load swapgeneric"); 1577 1578 (void) modloadonly("sys", "lbl_edition"); 1579 1580 dispinit(); 1581 1582 /* 1583 * Infer meanings to the members of the idprom buffer. 1584 */ 1585 parse_idprom(); 1586 1587 /* Read cluster configuration data. */ 1588 clconf_init(); 1589 1590 setup_ddi(); 1591 1592 /* 1593 * Lets take this opportunity to load the root device. 1594 */ 1595 if (loadrootmodules() != 0) 1596 debug_enter("Can't load the root filesystem"); 1597 1598 /* 1599 * Load tod driver module for the tod part found on this system. 1600 * Recompute the cpu frequency/delays based on tod as tod part 1601 * tends to keep time more accurately. 1602 */ 1603 if (&load_tod_module) 1604 load_tod_module(); 1605 1606 /* 1607 * Allow platforms to load modules which might 1608 * be needed after bootops are gone. 1609 */ 1610 if (&load_platform_modules) 1611 load_platform_modules(); 1612 1613 setcpudelay(); 1614 1615 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1616 &boot_physavail, &boot_physavail_len, 1617 &boot_virtavail, &boot_virtavail_len); 1618 1619 /* 1620 * Calculation and allocation of hmeblks needed to remap 1621 * the memory allocated by PROM till now. 1622 * Overestimate the number of hblk1 elements by assuming 1623 * worst case of TTE64K mappings. 1624 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1625 */ 1626 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1627 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1628 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1629 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1630 1631 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1632 1633 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1634 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1635 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1636 1637 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1638 1639 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1640 nhblk8 = 0; 1641 while (bop_alloc_pages > 1) { 1642 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1643 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1644 bop_alloc_pages *= hme8blk_sz; 1645 bop_alloc_pages = btopr(bop_alloc_pages); 1646 } 1647 nhblk8 += 2; 1648 1649 /* 1650 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1651 * boundary, the number of hblk8's needed to map the entries in the 1652 * boot_virtavail list needs to be adjusted to take this into 1653 * consideration. Thus, we need to add additional hblk8's since it 1654 * is possible that an hblk8 will not have all 8 slots used due to 1655 * alignment constraints. Since there were boot_virtavail_len entries 1656 * in that list, we need to add that many hblk8's to the number 1657 * already calculated to make sure we don't underestimate. 1658 */ 1659 nhblk8 += boot_virtavail_len; 1660 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1661 1662 /* Allocate in pagesize chunks */ 1663 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1664 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1665 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1666 } 1667 1668 static void 1669 startup_bop_gone(void) 1670 { 1671 1672 /* 1673 * Destroy the MD initialized at startup 1674 * The startup initializes the MD framework 1675 * using prom and BOP alloc free it now. 1676 */ 1677 mach_descrip_startup_fini(); 1678 1679 /* 1680 * We're done with prom allocations. 1681 */ 1682 bop_fini(); 1683 1684 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1685 &boot_physavail, &boot_physavail_len, 1686 &boot_virtavail, &boot_virtavail_len); 1687 1688 /* 1689 * setup physically contiguous area twice as large as the ecache. 1690 * this is used while doing displacement flush of ecaches 1691 */ 1692 if (&ecache_flush_address) { 1693 ecache_flushaddr = ecache_flush_address(); 1694 if (ecache_flushaddr == (uint64_t)-1) { 1695 cmn_err(CE_PANIC, 1696 "startup: no memory to set ecache_flushaddr"); 1697 } 1698 } 1699 1700 /* 1701 * Virtual available next. 1702 */ 1703 ASSERT(virt_avail != NULL); 1704 memlist_free_list(virt_avail); 1705 virt_avail = memlist; 1706 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1707 1708 } 1709 1710 1711 /* 1712 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1713 * allocations have been performed. We can't call it in startup_bop_gone 1714 * since later operations can cause obp to allocate more memory. 1715 */ 1716 void 1717 startup_fixup_physavail(void) 1718 { 1719 struct memlist *cur; 1720 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1721 1722 PRM_DEBUG(kmem64_overmap_size); 1723 1724 /* 1725 * take the most current snapshot we can by calling mem-update 1726 */ 1727 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1728 &boot_physavail, &boot_physavail_len, 1729 &boot_virtavail, &boot_virtavail_len); 1730 1731 /* 1732 * Copy phys_avail list, again. 1733 * Both the kernel/boot and the prom have been allocating 1734 * from the original list we copied earlier. 1735 */ 1736 cur = memlist; 1737 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1738 1739 /* 1740 * Add any unused kmem64 memory from overmapped page 1741 * (Note: va_to_pa does not work for kmem64_end) 1742 */ 1743 if (kmem64_overmap_size) { 1744 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1745 kmem64_overmap_size, &memlist, &cur); 1746 } 1747 1748 /* 1749 * Add any extra memory after e_data we added to the phys_avail list 1750 * back to the old list. 1751 */ 1752 if (ndata_remain_sz >= MMU_PAGESIZE) 1753 memlist_add(va_to_pa(nalloc_base), 1754 (uint64_t)ndata_remain_sz, &memlist, &cur); 1755 1756 /* 1757 * There isn't any bounds checking on the memlist area 1758 * so ensure it hasn't overgrown. 1759 */ 1760 if ((caddr_t)memlist > (caddr_t)memlist_end) 1761 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1762 1763 /* 1764 * The kernel removes the pages that were allocated for it from 1765 * the freelist, but we now have to find any -extra- pages that 1766 * the prom has allocated for it's own book-keeping, and remove 1767 * them from the freelist too. sigh. 1768 */ 1769 sync_memlists(phys_avail, cur); 1770 1771 ASSERT(phys_avail != NULL); 1772 1773 old_phys_avail = phys_avail; 1774 phys_avail = cur; 1775 } 1776 1777 void 1778 update_kcage_ranges(uint64_t addr, uint64_t len) 1779 { 1780 pfn_t base = btop(addr); 1781 pgcnt_t num = btop(len); 1782 int rv; 1783 1784 rv = kcage_range_add(base, num, kcage_startup_dir); 1785 1786 if (rv == ENOMEM) { 1787 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1788 (len == 0 ? 0 : BYTES_TO_MB(len))); 1789 } else if (rv != 0) { 1790 /* catch this in debug kernels */ 1791 ASSERT(0); 1792 1793 cmn_err(CE_WARN, "unexpected kcage_range_add" 1794 " return value %d", rv); 1795 } 1796 } 1797 1798 static void 1799 startup_vm(void) 1800 { 1801 size_t i; 1802 struct segmap_crargs a; 1803 struct segkpm_crargs b; 1804 1805 uint64_t avmem; 1806 caddr_t va; 1807 pgcnt_t max_phys_segkp; 1808 int mnode; 1809 1810 extern int use_brk_lpg, use_stk_lpg; 1811 1812 /* 1813 * get prom's mappings, create hments for them and switch 1814 * to the kernel context. 1815 */ 1816 hat_kern_setup(); 1817 1818 /* 1819 * Take over trap table 1820 */ 1821 setup_trap_table(); 1822 1823 /* 1824 * Install the va>tte handler, so that the prom can handle 1825 * misses and understand the kernel table layout in case 1826 * we need call into the prom. 1827 */ 1828 install_va_to_tte(); 1829 1830 /* 1831 * Set a flag to indicate that the tba has been taken over. 1832 */ 1833 tba_taken_over = 1; 1834 1835 /* initialize MMU primary context register */ 1836 mmu_init_kcontext(); 1837 1838 /* 1839 * The boot cpu can now take interrupts, x-calls, x-traps 1840 */ 1841 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1842 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1843 1844 /* 1845 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1846 */ 1847 tbr_wr_addr_inited = 1; 1848 1849 /* 1850 * Initialize VM system, and map kernel address space. 1851 */ 1852 kvm_init(); 1853 1854 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1855 if (kernel_cage_enable) { 1856 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1857 } 1858 memlist_free_list(old_phys_avail); 1859 1860 /* 1861 * If the following is true, someone has patched 1862 * phsymem to be less than the number of pages that 1863 * the system actually has. Remove pages until system 1864 * memory is limited to the requested amount. Since we 1865 * have allocated page structures for all pages, we 1866 * correct the amount of memory we want to remove 1867 * by the size of the memory used to hold page structures 1868 * for the non-used pages. 1869 */ 1870 if (physmem + ramdisk_npages < npages) { 1871 pgcnt_t diff, off; 1872 struct page *pp; 1873 struct seg kseg; 1874 1875 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1876 1877 off = 0; 1878 diff = npages - (physmem + ramdisk_npages); 1879 diff -= mmu_btopr(diff * sizeof (struct page)); 1880 kseg.s_as = &kas; 1881 while (diff--) { 1882 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1883 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1884 &kseg, (caddr_t)off); 1885 if (pp == NULL) 1886 cmn_err(CE_PANIC, "limited physmem too much!"); 1887 page_io_unlock(pp); 1888 page_downgrade(pp); 1889 availrmem--; 1890 off += MMU_PAGESIZE; 1891 } 1892 } 1893 1894 /* 1895 * When printing memory, show the total as physmem less 1896 * that stolen by a debugger. 1897 */ 1898 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1899 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1900 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1901 1902 avmem = (uint64_t)freemem << PAGESHIFT; 1903 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1904 1905 /* 1906 * For small memory systems disable automatic large pages. 1907 */ 1908 if (physmem < privm_lpg_min_physmem) { 1909 use_brk_lpg = 0; 1910 use_stk_lpg = 0; 1911 } 1912 1913 /* 1914 * Perform platform specific freelist processing 1915 */ 1916 if (&plat_freelist_process) { 1917 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1918 if (mem_node_config[mnode].exists) 1919 plat_freelist_process(mnode); 1920 } 1921 1922 /* 1923 * Initialize the segkp segment type. We position it 1924 * after the configured tables and buffers (whose end 1925 * is given by econtig) and before V_WKBASE_ADDR. 1926 * Also in this area is segkmap (size SEGMAPSIZE). 1927 */ 1928 1929 /* XXX - cache alignment? */ 1930 va = (caddr_t)SEGKPBASE; 1931 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1932 1933 max_phys_segkp = (physmem * 2); 1934 1935 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1936 segkpsize = btop(SEGKPDEFSIZE); 1937 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1938 "segkpsize has been reset to %ld pages", segkpsize); 1939 } 1940 1941 i = ptob(MIN(segkpsize, max_phys_segkp)); 1942 1943 rw_enter(&kas.a_lock, RW_WRITER); 1944 if (seg_attach(&kas, va, i, segkp) < 0) 1945 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1946 if (segkp_create(segkp) != 0) 1947 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1948 rw_exit(&kas.a_lock); 1949 1950 /* 1951 * kpm segment 1952 */ 1953 segmap_kpm = kpm_enable && 1954 segmap_kpm && PAGESIZE == MAXBSIZE; 1955 1956 if (kpm_enable) { 1957 rw_enter(&kas.a_lock, RW_WRITER); 1958 1959 /* 1960 * The segkpm virtual range range is larger than the 1961 * actual physical memory size and also covers gaps in 1962 * the physical address range for the following reasons: 1963 * . keep conversion between segkpm and physical addresses 1964 * simple, cheap and unambiguous. 1965 * . avoid extension/shrink of the the segkpm in case of DR. 1966 * . avoid complexity for handling of virtual addressed 1967 * caches, segkpm and the regular mapping scheme must be 1968 * kept in sync wrt. the virtual color of mapped pages. 1969 * Any accesses to virtual segkpm ranges not backed by 1970 * physical memory will fall through the memseg pfn hash 1971 * and will be handled in segkpm_fault. 1972 * Additional kpm_size spaces needed for vac alias prevention. 1973 */ 1974 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1975 segkpm) < 0) 1976 cmn_err(CE_PANIC, "cannot attach segkpm"); 1977 1978 b.prot = PROT_READ | PROT_WRITE; 1979 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1980 1981 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1982 panic("segkpm_create segkpm"); 1983 1984 rw_exit(&kas.a_lock); 1985 1986 mach_kpm_init(); 1987 } 1988 1989 if (!segzio_fromheap) { 1990 size_t size; 1991 size_t physmem_b = mmu_ptob(physmem); 1992 1993 /* size is in bytes, segziosize is in pages */ 1994 if (segziosize == 0) { 1995 size = physmem_b; 1996 } else { 1997 size = mmu_ptob(segziosize); 1998 } 1999 2000 if (size < SEGZIOMINSIZE) { 2001 size = SEGZIOMINSIZE; 2002 } else if (size > SEGZIOMAXSIZE) { 2003 size = SEGZIOMAXSIZE; 2004 /* 2005 * On 64-bit x86, we only have 2TB of KVA. This exists 2006 * for parity with x86. 2007 * 2008 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2009 * doesn't consume all of KVA. However, if we have a 2010 * system that has more thant 512gb of physical memory, 2011 * we can actually consume about half of the difference 2012 * between 512gb and the rest of the available physical 2013 * memory. 2014 */ 2015 if (physmem_b > SEGZIOMAXSIZE) { 2016 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2017 } 2018 } 2019 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2020 /* put the base of the ZIO segment after the kpm segment */ 2021 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2022 PRM_DEBUG(segziosize); 2023 PRM_DEBUG(segzio_base); 2024 2025 /* 2026 * On some platforms, kvm_init is called after the kpm 2027 * sizes have been determined. On SPARC, kvm_init is called 2028 * before, so we have to attach the kzioseg after kvm is 2029 * initialized, otherwise we'll try to allocate from the boot 2030 * area since the kernel heap hasn't yet been configured. 2031 */ 2032 rw_enter(&kas.a_lock, RW_WRITER); 2033 2034 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2035 &kzioseg); 2036 (void) segkmem_zio_create(&kzioseg); 2037 2038 /* create zio area covering new segment */ 2039 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2040 2041 rw_exit(&kas.a_lock); 2042 } 2043 2044 2045 /* 2046 * Now create generic mapping segment. This mapping 2047 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2048 * virtual address is greater than the amount of free 2049 * memory that is available, then we trim back the 2050 * segment size to that amount 2051 */ 2052 va = (caddr_t)SEGMAPBASE; 2053 2054 /* 2055 * 1201049: segkmap base address must be MAXBSIZE aligned 2056 */ 2057 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2058 2059 /* 2060 * Set size of segmap to percentage of freemem at boot, 2061 * but stay within the allowable range 2062 * Note we take percentage before converting from pages 2063 * to bytes to avoid an overflow on 32-bit kernels. 2064 */ 2065 i = mmu_ptob((freemem * segmap_percent) / 100); 2066 2067 if (i < MINMAPSIZE) 2068 i = MINMAPSIZE; 2069 2070 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2071 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2072 2073 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2074 2075 rw_enter(&kas.a_lock, RW_WRITER); 2076 if (seg_attach(&kas, va, i, segkmap) < 0) 2077 cmn_err(CE_PANIC, "cannot attach segkmap"); 2078 2079 a.prot = PROT_READ | PROT_WRITE; 2080 a.shmsize = shm_alignment; 2081 a.nfreelist = 0; /* use segmap driver defaults */ 2082 2083 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2084 panic("segmap_create segkmap"); 2085 rw_exit(&kas.a_lock); 2086 2087 segdev_init(); 2088 } 2089 2090 static void 2091 startup_end(void) 2092 { 2093 if ((caddr_t)memlist > (caddr_t)memlist_end) 2094 panic("memlist overflow 2"); 2095 memlist_free_block((caddr_t)memlist, 2096 ((caddr_t)memlist_end - (caddr_t)memlist)); 2097 memlist = NULL; 2098 2099 /* enable page_relocation since OBP is now done */ 2100 page_relocate_ready = 1; 2101 2102 /* 2103 * Perform tasks that get done after most of the VM 2104 * initialization has been done but before the clock 2105 * and other devices get started. 2106 */ 2107 kern_setup1(); 2108 2109 /* 2110 * Perform CPC initialization for this CPU. 2111 */ 2112 kcpc_hw_init(); 2113 2114 /* 2115 * Intialize the VM arenas for allocating physically 2116 * contiguus memory chunk for interrupt queues snd 2117 * allocate/register boot cpu's queues, if any and 2118 * allocate dump buffer for sun4v systems to store 2119 * extra crash information during crash dump 2120 */ 2121 contig_mem_init(); 2122 mach_descrip_init(); 2123 2124 if (cpu_intrq_setup(CPU)) { 2125 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2126 } 2127 cpu_intrq_register(CPU); 2128 mach_htraptrace_setup(CPU->cpu_id); 2129 mach_htraptrace_configure(CPU->cpu_id); 2130 mach_dump_buffer_init(); 2131 2132 /* 2133 * Initialize interrupt related stuff 2134 */ 2135 cpu_intr_alloc(CPU, NINTR_THREADS); 2136 2137 (void) splzs(); /* allow hi clock ints but not zs */ 2138 2139 /* 2140 * Initialize errors. 2141 */ 2142 error_init(); 2143 2144 /* 2145 * Note that we may have already used kernel bcopy before this 2146 * point - but if you really care about this, adb the use_hw_* 2147 * variables to 0 before rebooting. 2148 */ 2149 mach_hw_copy_limit(); 2150 2151 /* 2152 * Install the "real" preemption guards before DDI services 2153 * are available. 2154 */ 2155 (void) prom_set_preprom(kern_preprom); 2156 (void) prom_set_postprom(kern_postprom); 2157 CPU->cpu_m.mutex_ready = 1; 2158 2159 /* 2160 * Initialize segnf (kernel support for non-faulting loads). 2161 */ 2162 segnf_init(); 2163 2164 /* 2165 * Configure the root devinfo node. 2166 */ 2167 configure(); /* set up devices */ 2168 mach_cpu_halt_idle(); 2169 } 2170 2171 2172 void 2173 post_startup(void) 2174 { 2175 #ifdef PTL1_PANIC_DEBUG 2176 extern void init_ptl1_thread(void); 2177 #endif /* PTL1_PANIC_DEBUG */ 2178 extern void abort_sequence_init(void); 2179 2180 /* 2181 * Set the system wide, processor-specific flags to be passed 2182 * to userland via the aux vector for performance hints and 2183 * instruction set extensions. 2184 */ 2185 bind_hwcap(); 2186 2187 /* 2188 * Startup memory scrubber (if any) 2189 */ 2190 mach_memscrub(); 2191 2192 /* 2193 * Allocate soft interrupt to handle abort sequence. 2194 */ 2195 abort_sequence_init(); 2196 2197 /* 2198 * Configure the rest of the system. 2199 * Perform forceloading tasks for /etc/system. 2200 */ 2201 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2202 /* 2203 * ON4.0: Force /proc module in until clock interrupt handle fixed 2204 * ON4.0: This must be fixed or restated in /etc/systems. 2205 */ 2206 (void) modload("fs", "procfs"); 2207 2208 /* load machine class specific drivers */ 2209 load_mach_drivers(); 2210 2211 /* load platform specific drivers */ 2212 if (&load_platform_drivers) 2213 load_platform_drivers(); 2214 2215 /* load vis simulation module, if we are running w/fpu off */ 2216 if (!fpu_exists) { 2217 if (modload("misc", "vis") == -1) 2218 halt("Can't load vis"); 2219 } 2220 2221 mach_fpras(); 2222 2223 maxmem = freemem; 2224 2225 #ifdef PTL1_PANIC_DEBUG 2226 init_ptl1_thread(); 2227 #endif /* PTL1_PANIC_DEBUG */ 2228 } 2229 2230 #ifdef PTL1_PANIC_DEBUG 2231 int ptl1_panic_test = 0; 2232 int ptl1_panic_xc_one_test = 0; 2233 int ptl1_panic_xc_all_test = 0; 2234 int ptl1_panic_xt_one_test = 0; 2235 int ptl1_panic_xt_all_test = 0; 2236 kthread_id_t ptl1_thread_p = NULL; 2237 kcondvar_t ptl1_cv; 2238 kmutex_t ptl1_mutex; 2239 int ptl1_recurse_count_threshold = 0x40; 2240 int ptl1_recurse_trap_threshold = 0x3d; 2241 extern void ptl1_recurse(int, int); 2242 extern void ptl1_panic_xt(int, int); 2243 2244 /* 2245 * Called once per second by timeout() to wake up 2246 * the ptl1_panic thread to see if it should cause 2247 * a trap to the ptl1_panic() code. 2248 */ 2249 /* ARGSUSED */ 2250 static void 2251 ptl1_wakeup(void *arg) 2252 { 2253 mutex_enter(&ptl1_mutex); 2254 cv_signal(&ptl1_cv); 2255 mutex_exit(&ptl1_mutex); 2256 } 2257 2258 /* 2259 * ptl1_panic cross call function: 2260 * Needed because xc_one() and xc_some() can pass 2261 * 64 bit args but ptl1_recurse() expects ints. 2262 */ 2263 static void 2264 ptl1_panic_xc(void) 2265 { 2266 ptl1_recurse(ptl1_recurse_count_threshold, 2267 ptl1_recurse_trap_threshold); 2268 } 2269 2270 /* 2271 * The ptl1 thread waits for a global flag to be set 2272 * and uses the recurse thresholds to set the stack depth 2273 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2274 * or indirectly via the cross call and cross trap functions. 2275 * 2276 * This is useful testing stack overflows and normal 2277 * ptl1_panic() states with a know stack frame. 2278 * 2279 * ptl1_recurse() is an asm function in ptl1_panic.s that 2280 * sets the {In, Local, Out, and Global} registers to a 2281 * know state on the stack and just prior to causing a 2282 * test ptl1_panic trap. 2283 */ 2284 static void 2285 ptl1_thread(void) 2286 { 2287 mutex_enter(&ptl1_mutex); 2288 while (ptl1_thread_p) { 2289 cpuset_t other_cpus; 2290 int cpu_id; 2291 int my_cpu_id; 2292 int target_cpu_id; 2293 int target_found; 2294 2295 if (ptl1_panic_test) { 2296 ptl1_recurse(ptl1_recurse_count_threshold, 2297 ptl1_recurse_trap_threshold); 2298 } 2299 2300 /* 2301 * Find potential targets for x-call and x-trap, 2302 * if any exist while preempt is disabled we 2303 * start a ptl1_panic if requested via a 2304 * globals. 2305 */ 2306 kpreempt_disable(); 2307 my_cpu_id = CPU->cpu_id; 2308 other_cpus = cpu_ready_set; 2309 CPUSET_DEL(other_cpus, CPU->cpu_id); 2310 target_found = 0; 2311 if (!CPUSET_ISNULL(other_cpus)) { 2312 /* 2313 * Pick the first one 2314 */ 2315 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2316 if (cpu_id == my_cpu_id) 2317 continue; 2318 2319 if (CPU_XCALL_READY(cpu_id)) { 2320 target_cpu_id = cpu_id; 2321 target_found = 1; 2322 break; 2323 } 2324 } 2325 ASSERT(target_found); 2326 2327 if (ptl1_panic_xc_one_test) { 2328 xc_one(target_cpu_id, 2329 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2330 } 2331 if (ptl1_panic_xc_all_test) { 2332 xc_some(other_cpus, 2333 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2334 } 2335 if (ptl1_panic_xt_one_test) { 2336 xt_one(target_cpu_id, 2337 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2338 } 2339 if (ptl1_panic_xt_all_test) { 2340 xt_some(other_cpus, 2341 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2342 } 2343 } 2344 kpreempt_enable(); 2345 (void) timeout(ptl1_wakeup, NULL, hz); 2346 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2347 } 2348 mutex_exit(&ptl1_mutex); 2349 } 2350 2351 /* 2352 * Called during early startup to create the ptl1_thread 2353 */ 2354 void 2355 init_ptl1_thread(void) 2356 { 2357 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2358 &p0, TS_RUN, 0); 2359 } 2360 #endif /* PTL1_PANIC_DEBUG */ 2361 2362 2363 static void 2364 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2365 { 2366 struct memlist *new; 2367 2368 new = *memlistp; 2369 new->address = start; 2370 new->size = len; 2371 *memlistp = new + 1; 2372 } 2373 2374 /* 2375 * Add to a memory list. 2376 * start = start of new memory segment 2377 * len = length of new memory segment in bytes 2378 * memlistp = pointer to array of available memory segment structures 2379 * curmemlistp = memory list to which to add segment. 2380 */ 2381 static void 2382 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2383 struct memlist **curmemlistp) 2384 { 2385 struct memlist *new = *memlistp; 2386 2387 memlist_new(start, len, memlistp); 2388 memlist_insert(new, curmemlistp); 2389 } 2390 2391 static int 2392 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2393 { 2394 int nseg; 2395 size_t memseg_sz; 2396 struct memseg *msp; 2397 2398 /* 2399 * The memseg list is for the chunks of physical memory that 2400 * will be managed by the vm system. The number calculated is 2401 * a guess as boot may fragment it more when memory allocations 2402 * are made before kphysm_init(). 2403 */ 2404 memseg_sz = (avail + 10) * sizeof (struct memseg); 2405 memseg_sz = roundup(memseg_sz, PAGESIZE); 2406 nseg = memseg_sz / sizeof (struct memseg); 2407 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2408 if (msp == NULL) 2409 return (1); 2410 PRM_DEBUG(memseg_free); 2411 2412 while (nseg--) { 2413 msp->next = memseg_free; 2414 memseg_free = msp; 2415 msp++; 2416 } 2417 return (0); 2418 } 2419 2420 /* 2421 * In the case of architectures that support dynamic addition of 2422 * memory at run-time there are two cases where memsegs need to 2423 * be initialized and added to the memseg list. 2424 * 1) memsegs that are constructed at startup. 2425 * 2) memsegs that are constructed at run-time on 2426 * hot-plug capable architectures. 2427 * This code was originally part of the function kphysm_init(). 2428 */ 2429 2430 static void 2431 memseg_list_add(struct memseg *memsegp) 2432 { 2433 struct memseg **prev_memsegp; 2434 pgcnt_t num; 2435 2436 /* insert in memseg list, decreasing number of pages order */ 2437 2438 num = MSEG_NPAGES(memsegp); 2439 2440 for (prev_memsegp = &memsegs; *prev_memsegp; 2441 prev_memsegp = &((*prev_memsegp)->next)) { 2442 if (num > MSEG_NPAGES(*prev_memsegp)) 2443 break; 2444 } 2445 2446 memsegp->next = *prev_memsegp; 2447 *prev_memsegp = memsegp; 2448 2449 if (kpm_enable) { 2450 memsegp->nextpa = (memsegp->next) ? 2451 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2452 2453 if (prev_memsegp != &memsegs) { 2454 struct memseg *msp; 2455 msp = (struct memseg *)((caddr_t)prev_memsegp - 2456 offsetof(struct memseg, next)); 2457 msp->nextpa = va_to_pa(memsegp); 2458 } else { 2459 memsegspa = va_to_pa(memsegs); 2460 } 2461 } 2462 } 2463 2464 /* 2465 * PSM add_physmem_cb(). US-II and newer processors have some 2466 * flavor of the prefetch capability implemented. We exploit 2467 * this capability for optimum performance. 2468 */ 2469 #define PREFETCH_BYTES 64 2470 2471 void 2472 add_physmem_cb(page_t *pp, pfn_t pnum) 2473 { 2474 extern void prefetch_page_w(void *); 2475 2476 pp->p_pagenum = pnum; 2477 2478 /* 2479 * Prefetch one more page_t into E$. To prevent future 2480 * mishaps with the sizeof(page_t) changing on us, we 2481 * catch this on debug kernels if we can't bring in the 2482 * entire hpage with 2 PREFETCH_BYTES reads. See 2483 * also, sun4u/cpu/cpu_module.c 2484 */ 2485 /*LINTED*/ 2486 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2487 prefetch_page_w((char *)pp); 2488 } 2489 2490 /* 2491 * Find memseg with given pfn 2492 */ 2493 static struct memseg * 2494 memseg_find(pfn_t base, pfn_t *next) 2495 { 2496 struct memseg *seg; 2497 2498 if (next != NULL) 2499 *next = LONG_MAX; 2500 for (seg = memsegs; seg != NULL; seg = seg->next) { 2501 if (base >= seg->pages_base && base < seg->pages_end) 2502 return (seg); 2503 if (next != NULL && seg->pages_base > base && 2504 seg->pages_base < *next) 2505 *next = seg->pages_base; 2506 } 2507 return (NULL); 2508 } 2509 2510 extern struct vnode prom_ppages; 2511 2512 /* 2513 * Put page allocated by OBP on prom_ppages 2514 */ 2515 static void 2516 kphysm_erase(uint64_t addr, uint64_t len) 2517 { 2518 struct page *pp; 2519 struct memseg *seg; 2520 pfn_t base = btop(addr), next; 2521 pgcnt_t num = btop(len); 2522 2523 while (num != 0) { 2524 pgcnt_t off, left; 2525 2526 seg = memseg_find(base, &next); 2527 if (seg == NULL) { 2528 if (next == LONG_MAX) 2529 break; 2530 left = MIN(next - base, num); 2531 base += left, num -= left; 2532 continue; 2533 } 2534 off = base - seg->pages_base; 2535 pp = seg->pages + off; 2536 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2537 while (num != left) { 2538 /* 2539 * init it, lock it, and hashin on prom_pages vp. 2540 * 2541 * XXX vnode offsets on the prom_ppages vnode 2542 * are page numbers (gack) for >32 bit 2543 * physical memory machines. 2544 */ 2545 add_physmem_cb(pp, base); 2546 if (page_trylock(pp, SE_EXCL) == 0) 2547 cmn_err(CE_PANIC, "prom page locked"); 2548 (void) page_hashin(pp, &prom_ppages, 2549 (offset_t)base, NULL); 2550 (void) page_pp_lock(pp, 0, 1); 2551 pp++, base++, num--; 2552 } 2553 } 2554 } 2555 2556 static page_t *ppnext; 2557 static pgcnt_t ppleft; 2558 2559 static void *kpm_ppnext; 2560 static pgcnt_t kpm_ppleft; 2561 2562 /* 2563 * Create a memseg 2564 */ 2565 static void 2566 kphysm_memseg(uint64_t addr, uint64_t len) 2567 { 2568 pfn_t base = btop(addr); 2569 pgcnt_t num = btop(len); 2570 struct memseg *seg; 2571 2572 seg = memseg_free; 2573 memseg_free = seg->next; 2574 ASSERT(seg != NULL); 2575 2576 seg->pages = ppnext; 2577 seg->epages = ppnext + num; 2578 seg->pages_base = base; 2579 seg->pages_end = base + num; 2580 ppnext += num; 2581 ppleft -= num; 2582 2583 if (kpm_enable) { 2584 pgcnt_t kpnum = ptokpmpr(num); 2585 2586 if (kpnum > kpm_ppleft) 2587 panic("kphysm_memseg: kpm_pp overflow"); 2588 seg->pagespa = va_to_pa(seg->pages); 2589 seg->epagespa = va_to_pa(seg->epages); 2590 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2591 seg->kpm_nkpmpgs = kpnum; 2592 /* 2593 * In the kpm_smallpage case, the kpm array 2594 * is 1-1 wrt the page array 2595 */ 2596 if (kpm_smallpages) { 2597 kpm_spage_t *kpm_pp = kpm_ppnext; 2598 2599 kpm_ppnext = kpm_pp + kpnum; 2600 seg->kpm_spages = kpm_pp; 2601 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2602 } else { 2603 kpm_page_t *kpm_pp = kpm_ppnext; 2604 2605 kpm_ppnext = kpm_pp + kpnum; 2606 seg->kpm_pages = kpm_pp; 2607 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2608 /* ASSERT no kpm overlaps */ 2609 ASSERT( 2610 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2611 ASSERT(memseg_find( 2612 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2613 } 2614 kpm_ppleft -= num; 2615 } 2616 2617 memseg_list_add(seg); 2618 } 2619 2620 /* 2621 * Add range to free list 2622 */ 2623 void 2624 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2625 { 2626 struct page *pp; 2627 struct memseg *seg; 2628 pfn_t base = btop(addr); 2629 pgcnt_t num = btop(len); 2630 2631 seg = memseg_find(base, NULL); 2632 ASSERT(seg != NULL); 2633 pp = seg->pages + (base - seg->pages_base); 2634 2635 if (reclaim) { 2636 struct page *rpp = pp; 2637 struct page *lpp = pp + num; 2638 2639 /* 2640 * page should be locked on prom_ppages 2641 * unhash and unlock it 2642 */ 2643 while (rpp < lpp) { 2644 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages); 2645 page_pp_unlock(rpp, 0, 1); 2646 page_hashout(rpp, NULL); 2647 page_unlock(rpp); 2648 rpp++; 2649 } 2650 } 2651 2652 /* 2653 * add_physmem() initializes the PSM part of the page 2654 * struct by calling the PSM back with add_physmem_cb(). 2655 * In addition it coalesces pages into larger pages as 2656 * it initializes them. 2657 */ 2658 add_physmem(pp, num, base); 2659 } 2660 2661 /* 2662 * kphysm_init() tackles the problem of initializing physical memory. 2663 */ 2664 static void 2665 kphysm_init(void) 2666 { 2667 struct memlist *pmem; 2668 2669 ASSERT(page_hash != NULL && page_hashsz != 0); 2670 2671 ppnext = pp_base; 2672 ppleft = npages; 2673 kpm_ppnext = kpm_pp_base; 2674 kpm_ppleft = kpm_npages; 2675 2676 /* 2677 * installed pages not on nopp_memlist go in memseg list 2678 */ 2679 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2680 2681 /* 2682 * Free the avail list 2683 */ 2684 for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 2685 kphysm_add(pmem->address, pmem->size, 0); 2686 2687 /* 2688 * Erase pages that aren't available 2689 */ 2690 diff_memlists(phys_install, phys_avail, kphysm_erase); 2691 2692 build_pfn_hash(); 2693 } 2694 2695 /* 2696 * Kernel VM initialization. 2697 * Assumptions about kernel address space ordering: 2698 * (1) gap (user space) 2699 * (2) kernel text 2700 * (3) kernel data/bss 2701 * (4) gap 2702 * (5) kernel data structures 2703 * (6) gap 2704 * (7) debugger (optional) 2705 * (8) monitor 2706 * (9) gap (possibly null) 2707 * (10) dvma 2708 * (11) devices 2709 */ 2710 static void 2711 kvm_init(void) 2712 { 2713 /* 2714 * Put the kernel segments in kernel address space. 2715 */ 2716 rw_enter(&kas.a_lock, RW_WRITER); 2717 as_avlinit(&kas); 2718 2719 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2720 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2721 (void) segkmem_create(&ktextseg); 2722 2723 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2724 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2725 (void) segkmem_create(&ktexthole); 2726 2727 (void) seg_attach(&kas, (caddr_t)valloc_base, 2728 (size_t)(econtig32 - valloc_base), &kvalloc); 2729 (void) segkmem_create(&kvalloc); 2730 2731 if (kmem64_base) { 2732 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2733 (size_t)(kmem64_end - kmem64_base), &kmem64); 2734 (void) segkmem_create(&kmem64); 2735 } 2736 2737 /* 2738 * We're about to map out /boot. This is the beginning of the 2739 * system resource management transition. We can no longer 2740 * call into /boot for I/O or memory allocations. 2741 */ 2742 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2743 (void) segkmem_create(&kvseg); 2744 hblk_alloc_dynamic = 1; 2745 2746 /* 2747 * we need to preallocate pages for DR operations before enabling large 2748 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2749 */ 2750 memseg_remap_init(); 2751 2752 /* at this point we are ready to use large page heap */ 2753 segkmem_heap_lp_init(); 2754 2755 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2756 &kvseg32); 2757 (void) segkmem_create(&kvseg32); 2758 2759 /* 2760 * Create a segment for the debugger. 2761 */ 2762 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2763 (void) segkmem_create(&kdebugseg); 2764 2765 rw_exit(&kas.a_lock); 2766 } 2767 2768 char obp_tte_str[] = 2769 "h# %x constant MMU_PAGESHIFT " 2770 "h# %x constant TTE8K " 2771 "h# %x constant SFHME_SIZE " 2772 "h# %x constant SFHME_TTE " 2773 "h# %x constant HMEBLK_TAG " 2774 "h# %x constant HMEBLK_NEXT " 2775 "h# %x constant HMEBLK_MISC " 2776 "h# %x constant HMEBLK_HME1 " 2777 "h# %x constant NHMENTS " 2778 "h# %x constant HBLK_SZMASK " 2779 "h# %x constant HBLK_RANGE_SHIFT " 2780 "h# %x constant HMEBP_HBLK " 2781 "h# %x constant HMEBLK_ENDPA " 2782 "h# %x constant HMEBUCKET_SIZE " 2783 "h# %x constant HTAG_SFMMUPSZ " 2784 "h# %x constant HTAG_BSPAGE_SHIFT " 2785 "h# %x constant HTAG_REHASH_SHIFT " 2786 "h# %x constant SFMMU_INVALID_SHMERID " 2787 "h# %x constant mmu_hashcnt " 2788 "h# %p constant uhme_hash " 2789 "h# %p constant khme_hash " 2790 "h# %x constant UHMEHASH_SZ " 2791 "h# %x constant KHMEHASH_SZ " 2792 "h# %p constant KCONTEXT " 2793 "h# %p constant KHATID " 2794 "h# %x constant ASI_MEM " 2795 "h# %x constant SOFTEXEC " 2796 "h# %x constant EXECPRM " 2797 2798 ": PHYS-X@ ( phys -- data ) " 2799 " ASI_MEM spacex@ " 2800 "; " 2801 2802 ": PHYS-W@ ( phys -- data ) " 2803 " ASI_MEM spacew@ " 2804 "; " 2805 2806 ": PHYS-L@ ( phys -- data ) " 2807 " ASI_MEM spaceL@ " 2808 "; " 2809 2810 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2811 " 3 * MMU_PAGESHIFT + " 2812 "; " 2813 2814 ": TTE_IS_VALID ( ttep -- flag ) " 2815 " PHYS-X@ 0< " 2816 "; " 2817 2818 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2819 " dup TTE8K = if " 2820 " drop HBLK_RANGE_SHIFT " 2821 " else " 2822 " TTE_PAGE_SHIFT " 2823 " then " 2824 "; " 2825 2826 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2827 " tuck >> swap MMU_PAGESHIFT - << " 2828 "; " 2829 2830 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2831 " >> over xor swap ( hash sfmmup ) " 2832 " KHATID <> if ( hash ) " 2833 " UHMEHASH_SZ and ( bucket ) " 2834 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2835 " else ( hash ) " 2836 " KHMEHASH_SZ and ( bucket ) " 2837 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2838 " then ( hmebp ) " 2839 "; " 2840 2841 ": HME_HASH_TABLE_SEARCH " 2842 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2843 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2844 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2845 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2846 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2847 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2848 " else " 2849 " hmeblk_next + phys-x@ false " 2850 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2851 " then " 2852 " else " 2853 " hmeblk_next + phys-x@ false " 2854 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2855 " then " 2856 " else " 2857 " drop 0 true " 2858 " then " 2859 " until r> drop " 2860 "; " 2861 2862 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2863 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2864 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2865 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2866 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2867 "; " 2868 2869 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2870 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2871 " TTE8K = if ( hmeblkp addr ) " 2872 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2873 " else ( hmeblkp addr ) " 2874 " drop 0 ( hmeblkp 0 ) " 2875 " then ( hmeblkp hme-index ) " 2876 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2877 " SFHME_TTE + ( ttep ) " 2878 "; " 2879 2880 ": unix-tte ( addr cnum -- false | tte-data true ) " 2881 " KCONTEXT = if ( addr ) " 2882 " KHATID ( addr khatid ) " 2883 " else ( addr ) " 2884 " drop false exit ( false ) " 2885 " then " 2886 " ( addr khatid ) " 2887 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2888 " 2dup swap i HME_HASH_SHIFT " 2889 "( addr sfmmup sfmmup addr hmeshift ) " 2890 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2891 " over i 4 pick " 2892 "( addr sfmmup hmebp sfmmup rehash addr ) " 2893 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2894 " HME_HASH_TABLE_SEARCH " 2895 "( addr sfmmup { null | hmeblkp } ) " 2896 " ?dup if ( addr sfmmup hmeblkp ) " 2897 " nip swap HBLK_TO_TTEP ( ttep ) " 2898 " dup TTE_IS_VALID if ( valid-ttep ) " 2899 " PHYS-X@ ( tte-data ) " 2900 " dup SOFTEXEC and 0> if ( tte-data ) " 2901 " SOFTEXEC - EXECPRM or ( tte-data ) " 2902 " then ( tte-data ) " 2903 " true ( tte-data true ) " 2904 " else ( invalid-tte ) " 2905 " drop false ( false ) " 2906 " then ( false | tte-data true ) " 2907 " unloop exit ( false | tte-data true ) " 2908 " then ( addr sfmmup ) " 2909 " loop ( addr sfmmup ) " 2910 " 2drop false ( false ) " 2911 "; " 2912 ; 2913 2914 void 2915 create_va_to_tte(void) 2916 { 2917 char *bp; 2918 extern int khmehash_num, uhmehash_num; 2919 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2920 2921 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2922 2923 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2924 2925 /* 2926 * Teach obp how to parse our sw ttes. 2927 */ 2928 (void) sprintf(bp, obp_tte_str, 2929 MMU_PAGESHIFT, 2930 TTE8K, 2931 sizeof (struct sf_hment), 2932 OFFSET(struct sf_hment, hme_tte), 2933 OFFSET(struct hme_blk, hblk_tag), 2934 OFFSET(struct hme_blk, hblk_nextpa), 2935 OFFSET(struct hme_blk, hblk_misc), 2936 OFFSET(struct hme_blk, hblk_hme), 2937 NHMENTS, 2938 HBLK_SZMASK, 2939 HBLK_RANGE_SHIFT, 2940 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2941 HMEBLK_ENDPA, 2942 sizeof (struct hmehash_bucket), 2943 HTAG_SFMMUPSZ, 2944 HTAG_BSPAGE_SHIFT, 2945 HTAG_REHASH_SHIFT, 2946 SFMMU_INVALID_SHMERID, 2947 mmu_hashcnt, 2948 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2949 (caddr_t)va_to_pa((caddr_t)khme_hash), 2950 UHMEHASH_SZ, 2951 KHMEHASH_SZ, 2952 KCONTEXT, 2953 KHATID, 2954 ASI_MEM, 2955 icache_is_coherent ? 0 : TTE_SOFTEXEC_INT, 2956 TTE_EXECPRM_INT); 2957 prom_interpret(bp, 0, 0, 0, 0, 0); 2958 2959 kobj_free(bp, MMU_PAGESIZE); 2960 } 2961 2962 void 2963 install_va_to_tte(void) 2964 { 2965 /* 2966 * advise prom that he can use unix-tte 2967 */ 2968 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2969 } 2970 2971 /* 2972 * Here we add "device-type=console" for /os-io node, for currently 2973 * our kernel console output only supports displaying text and 2974 * performing cursor-positioning operations (through kernel framebuffer 2975 * driver) and it doesn't support other functionalities required for a 2976 * standard "display" device as specified in 1275 spec. The main missing 2977 * interface defined by the 1275 spec is "draw-logo". 2978 * also see the comments above prom_stdout_is_framebuffer(). 2979 */ 2980 static char *create_node = 2981 "\" /\" find-device " 2982 "new-device " 2983 "\" os-io\" device-name " 2984 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 2985 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2986 " 2>r swap 2 2r> ['] $callback catch if " 2987 " 2drop 3drop 0 " 2988 " then " 2989 "; " 2990 ": read ( adr,len -- #read ) " 2991 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2992 " ( retN ... ret1 N ) " 2993 " ?dup if " 2994 " swap >r 1- 0 ?do drop loop r> " 2995 " else " 2996 " -2 " 2997 " then " 2998 "; " 2999 ": write ( adr,len -- #written ) " 3000 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3001 " ( retN ... ret1 N ) " 3002 " ?dup if " 3003 " swap >r 1- 0 ?do drop loop r> " 3004 " else " 3005 " 0 " 3006 " then " 3007 "; " 3008 ": poll-tty ( -- ) ; " 3009 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3010 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3011 ": cb-give/take ( $method -- ) " 3012 " 0 -rot ['] $callback catch ?dup if " 3013 " >r 2drop 2drop r> throw " 3014 " else " 3015 " 0 ?do drop loop " 3016 " then " 3017 "; " 3018 ": give ( -- ) \" exit-input\" cb-give/take ; " 3019 ": take ( -- ) \" enter-input\" cb-give/take ; " 3020 ": open ( -- ok? ) true ; " 3021 ": close ( -- ) ; " 3022 "finish-device " 3023 "device-end "; 3024 3025 /* 3026 * Create the OBP input/output node (FCode serial driver). 3027 * It is needed for both USB console keyboard and for 3028 * the kernel terminal emulator. It is too early to check for a 3029 * kernel console compatible framebuffer now, so we create this 3030 * so that we're ready if we need to enable kernel terminal emulation. 3031 * 3032 * When the USB software takes over the input device at the time 3033 * consconfig runs, OBP's stdin is redirected to this node. 3034 * Whenever the FORTH user interface is used after this switch, 3035 * the node will call back into the kernel for console input. 3036 * If a serial device such as ttya or a UART with a Type 5 keyboard 3037 * attached is used, OBP takes over the serial device when the system 3038 * goes to the debugger after the system is booted. This sharing 3039 * of the relatively simple serial device is difficult but possible. 3040 * Sharing the USB host controller is impossible due its complexity. 3041 * 3042 * Similarly to USB keyboard input redirection, after consconfig_dacf 3043 * configures a kernel console framebuffer as the standard output 3044 * device, OBP's stdout is switched to to vector through the 3045 * /os-io node into the kernel terminal emulator. 3046 */ 3047 static void 3048 startup_create_io_node(void) 3049 { 3050 prom_interpret(create_node, 0, 0, 0, 0, 0); 3051 } 3052 3053 3054 static void 3055 do_prom_version_check(void) 3056 { 3057 int i; 3058 pnode_t node; 3059 char buf[64]; 3060 static char drev[] = "Down-rev firmware detected%s\n" 3061 "\tPlease upgrade to the following minimum version:\n" 3062 "\t\t%s\n"; 3063 3064 i = prom_version_check(buf, sizeof (buf), &node); 3065 3066 if (i == PROM_VER64_OK) 3067 return; 3068 3069 if (i == PROM_VER64_UPGRADE) { 3070 cmn_err(CE_WARN, drev, "", buf); 3071 3072 #ifdef DEBUG 3073 prom_enter_mon(); /* Type 'go' to continue */ 3074 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3075 return; 3076 #else 3077 halt(0); 3078 #endif 3079 } 3080 3081 /* 3082 * The other possibility is that this is a server running 3083 * good firmware, but down-rev firmware was detected on at 3084 * least one other cpu board. We just complain if we see 3085 * that. 3086 */ 3087 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3088 } 3089 3090 3091 /* 3092 * Must be defined in platform dependent code. 3093 */ 3094 extern caddr_t modtext; 3095 extern size_t modtext_sz; 3096 extern caddr_t moddata; 3097 3098 #define HEAPTEXT_ARENA(addr) \ 3099 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3100 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3101 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3102 3103 #define HEAPTEXT_OVERSIZED(addr) \ 3104 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3105 3106 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3107 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3108 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3109 3110 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3111 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3112 kmutex_t texthole_lock; 3113 3114 char kern_bootargs[OBP_MAXPATHLEN]; 3115 char kern_bootfile[OBP_MAXPATHLEN]; 3116 3117 void 3118 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3119 { 3120 uintptr_t addr, limit; 3121 3122 addr = HEAPTEXT_BASE; 3123 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3124 3125 /* 3126 * Before we initialize the text_arena, we want to punch holes in the 3127 * underlying heaptext_arena. This guarantees that for any text 3128 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3129 */ 3130 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3131 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3132 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3133 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3134 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3135 } 3136 3137 /* 3138 * Allocate one page at the oversize to break up the text region 3139 * from the oversized region. 3140 */ 3141 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3142 (void *)limit, (void *)(limit + PAGESIZE), 3143 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3144 3145 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3146 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3147 heaptext_arena, 0, VM_SLEEP); 3148 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3149 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3150 } 3151 3152 caddr_t 3153 kobj_text_alloc(vmem_t *arena, size_t size) 3154 { 3155 caddr_t rval, better; 3156 3157 /* 3158 * First, try a sleeping allocation. 3159 */ 3160 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3161 3162 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3163 return (rval); 3164 3165 /* 3166 * We didn't get the area that we wanted. We're going to try to do an 3167 * allocation with explicit constraints. 3168 */ 3169 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3170 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3171 VM_NOSLEEP | VM_BESTFIT); 3172 3173 if (better != NULL) { 3174 /* 3175 * That worked. Free our first attempt and return. 3176 */ 3177 vmem_free(arena, rval, size); 3178 return (better); 3179 } 3180 3181 /* 3182 * That didn't work; we'll have to return our first attempt. 3183 */ 3184 return (rval); 3185 } 3186 3187 caddr_t 3188 kobj_texthole_alloc(caddr_t addr, size_t size) 3189 { 3190 int arena = HEAPTEXT_ARENA(addr); 3191 char c[30]; 3192 uintptr_t base; 3193 3194 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3195 /* 3196 * If this is an oversized allocation or it is allocated in 3197 * the nucleus data page, there is no text hole available for 3198 * it; return NULL. 3199 */ 3200 return (NULL); 3201 } 3202 3203 mutex_enter(&texthole_lock); 3204 3205 if (texthole_arena[arena] == NULL) { 3206 ASSERT(texthole_source[arena] == NULL); 3207 3208 if (arena == 0) { 3209 texthole_source[0] = vmem_create("module_text_holesrc", 3210 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3211 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3212 0, VM_SLEEP); 3213 } else { 3214 base = HEAPTEXT_BASE + 3215 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3216 3217 (void) snprintf(c, sizeof (c), 3218 "heaptext_holesrc_%d", arena); 3219 3220 texthole_source[arena] = vmem_create(c, (void *)base, 3221 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3222 0, VM_SLEEP); 3223 } 3224 3225 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3226 3227 texthole_arena[arena] = vmem_create(c, NULL, 0, 3228 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3229 texthole_source[arena], 0, VM_SLEEP); 3230 } 3231 3232 mutex_exit(&texthole_lock); 3233 3234 ASSERT(texthole_arena[arena] != NULL); 3235 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3236 return (vmem_alloc(texthole_arena[arena], size, 3237 VM_BESTFIT | VM_NOSLEEP)); 3238 } 3239 3240 void 3241 kobj_texthole_free(caddr_t addr, size_t size) 3242 { 3243 int arena = HEAPTEXT_ARENA(addr); 3244 3245 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3246 ASSERT(texthole_arena[arena] != NULL); 3247 vmem_free(texthole_arena[arena], addr, size); 3248 } 3249 3250 void 3251 release_bootstrap(void) 3252 { 3253 if (&cif_init) 3254 cif_init(); 3255 } 3256