1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern int cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 74 extern void mach_dump_buffer_init(void); 75 extern void mach_descrip_init(void); 76 extern void mach_descrip_startup_fini(void); 77 extern void mach_memscrub(void); 78 extern void mach_fpras(void); 79 extern void mach_cpu_halt_idle(void); 80 extern void mach_hw_copy_limit(void); 81 extern void load_mach_drivers(void); 82 extern void load_tod_module(void); 83 #pragma weak load_tod_module 84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata); 86 #pragma weak ndata_alloc_mmfsa 87 88 extern void cif_init(void); 89 #pragma weak cif_init 90 91 extern void parse_idprom(void); 92 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 93 extern void mem_config_init(void); 94 extern void memseg_remap_init(void); 95 96 extern void mach_kpm_init(void); 97 extern void pcf_init(); 98 extern int size_pse_array(pgcnt_t, int); 99 100 /* 101 * External Data: 102 */ 103 extern int vac_size; /* cache size in bytes */ 104 extern uint_t vac_mask; /* VAC alignment consistency mask */ 105 extern uint_t vac_colors; 106 107 /* 108 * Global Data Definitions: 109 */ 110 111 /* 112 * XXX - Don't port this to new architectures 113 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 114 * 'romp' has no use with a prom with an IEEE 1275 client interface. 115 * The driver doesn't use the value, but it depends on the symbol. 116 */ 117 void *romp; /* veritas driver won't load without romp 4154976 */ 118 /* 119 * Declare these as initialized data so we can patch them. 120 */ 121 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 122 pgcnt_t segkpsize = 123 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 124 uint_t segmap_percent = 12; /* Size of segmap segment */ 125 126 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 127 int vac_copyback = 1; 128 char *cache_mode = NULL; 129 int use_mix = 1; 130 int prom_debug = 0; 131 132 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 133 uint_t tba_taken_over = 0; 134 135 caddr_t s_text; /* start of kernel text segment */ 136 caddr_t e_text; /* end of kernel text segment */ 137 caddr_t s_data; /* start of kernel data segment */ 138 caddr_t e_data; /* end of kernel data segment */ 139 140 caddr_t modtext; /* beginning of module text */ 141 size_t modtext_sz; /* size of module text */ 142 caddr_t moddata; /* beginning of module data reserve */ 143 caddr_t e_moddata; /* end of module data reserve */ 144 145 /* 146 * End of first block of contiguous kernel in 32-bit virtual address space 147 */ 148 caddr_t econtig32; /* end of first blk of contiguous kernel */ 149 150 caddr_t ncbase; /* beginning of non-cached segment */ 151 caddr_t ncend; /* end of non-cached segment */ 152 153 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 154 caddr_t nalloc_base; /* beginning of nucleus allocation */ 155 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 156 caddr_t valloc_base; /* beginning of kvalloc segment */ 157 158 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 159 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 160 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 161 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 162 int kmem64_szc; /* page size code */ 163 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 164 165 uintptr_t shm_alignment; /* VAC address consistency modulus */ 166 struct memlist *phys_install; /* Total installed physical memory */ 167 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 168 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 169 struct memlist *nopp_list; /* pages with no backing page structs */ 170 struct memlist ndata; /* memlist of nucleus allocatable memory */ 171 int memexp_flag; /* memory expansion card flag */ 172 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 173 pgcnt_t obp_pages; /* Physical pages used by OBP */ 174 175 /* 176 * VM data structures 177 */ 178 long page_hashsz; /* Size of page hash table (power of two) */ 179 struct page *pp_base; /* Base of system page struct array */ 180 size_t pp_sz; /* Size in bytes of page struct array */ 181 struct page **page_hash; /* Page hash table */ 182 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 183 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 184 int pse_shift; /* log2(pse_table_size) */ 185 struct seg ktextseg; /* Segment used for kernel executable image */ 186 struct seg kvalloc; /* Segment used for "valloc" mapping */ 187 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 188 struct seg ktexthole; /* Segment used for nucleus text hole */ 189 struct seg kmapseg; /* Segment used for generic kernel mappings */ 190 struct seg kpmseg; /* Segment used for physical mapping */ 191 struct seg kdebugseg; /* Segment used for the kernel debugger */ 192 193 void *kpm_pp_base; /* Base of system kpm_page array */ 194 size_t kpm_pp_sz; /* Size of system kpm_page array */ 195 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 196 197 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 198 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 199 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 200 201 int segzio_fromheap = 0; /* zio allocations occur from heap */ 202 caddr_t segzio_base; /* Base address of segzio */ 203 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 204 205 /* 206 * debugger pages (if allocated) 207 */ 208 struct vnode kdebugvp; 209 210 /* 211 * VA range available to the debugger 212 */ 213 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 214 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 215 216 /* 217 * Segment for relocated kernel structures in 64-bit large RAM kernels 218 */ 219 struct seg kmem64; 220 221 struct memseg *memseg_free; 222 223 struct vnode unused_pages_vp; 224 225 /* 226 * VM data structures allocated early during boot. 227 */ 228 size_t pagehash_sz; 229 uint64_t memlist_sz; 230 231 char tbr_wr_addr_inited = 0; 232 233 caddr_t mpo_heap32_buf = NULL; 234 size_t mpo_heap32_bufsz = 0; 235 236 /* 237 * Static Routines: 238 */ 239 static int ndata_alloc_memseg(struct memlist *, size_t); 240 static void memlist_new(uint64_t, uint64_t, struct memlist **); 241 static void memlist_add(uint64_t, uint64_t, 242 struct memlist **, struct memlist **); 243 static void kphysm_init(void); 244 static void kvm_init(void); 245 static void install_kmem64_tte(void); 246 247 static void startup_init(void); 248 static void startup_memlist(void); 249 static void startup_modules(void); 250 static void startup_bop_gone(void); 251 static void startup_vm(void); 252 static void startup_end(void); 253 static void setup_cage_params(void); 254 static void startup_create_io_node(void); 255 256 static pgcnt_t npages; 257 static struct memlist *memlist; 258 void *memlist_end; 259 260 static pgcnt_t bop_alloc_pages; 261 static caddr_t hblk_base; 262 uint_t hblk_alloc_dynamic = 0; 263 uint_t hblk1_min = H1MIN; 264 265 266 /* 267 * Hooks for unsupported platforms and down-rev firmware 268 */ 269 int iam_positron(void); 270 #pragma weak iam_positron 271 static void do_prom_version_check(void); 272 273 /* 274 * After receiving a thermal interrupt, this is the number of seconds 275 * to delay before shutting off the system, assuming 276 * shutdown fails. Use /etc/system to change the delay if this isn't 277 * large enough. 278 */ 279 int thermal_powerdown_delay = 1200; 280 281 /* 282 * Used to hold off page relocations into the cage until OBP has completed 283 * its boot-time handoff of its resources to the kernel. 284 */ 285 int page_relocate_ready = 0; 286 287 /* 288 * Indicate if kmem64 allocation was done in small chunks 289 */ 290 int kmem64_smchunks = 0; 291 292 /* 293 * Enable some debugging messages concerning memory usage... 294 */ 295 #ifdef DEBUGGING_MEM 296 static int debugging_mem; 297 static void 298 printmemlist(char *title, struct memlist *list) 299 { 300 if (!debugging_mem) 301 return; 302 303 printf("%s\n", title); 304 305 while (list) { 306 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 307 (uint32_t)(list->address >> 32), (uint32_t)list->address, 308 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 309 list = list->next; 310 } 311 } 312 313 void 314 printmemseg(struct memseg *memseg) 315 { 316 if (!debugging_mem) 317 return; 318 319 printf("memseg\n"); 320 321 while (memseg) { 322 prom_printf("\tpage = 0x%p, epage = 0x%p, " 323 "pfn = 0x%x, epfn = 0x%x\n", 324 memseg->pages, memseg->epages, 325 memseg->pages_base, memseg->pages_end); 326 memseg = memseg->next; 327 } 328 } 329 330 #define debug_pause(str) halt((str)) 331 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 332 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 333 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 334 #define MPRINTF3(str, a, b, c) \ 335 if (debugging_mem) prom_printf((str), (a), (b), (c)) 336 #else /* DEBUGGING_MEM */ 337 #define MPRINTF(str) 338 #define MPRINTF1(str, a) 339 #define MPRINTF2(str, a, b) 340 #define MPRINTF3(str, a, b, c) 341 #endif /* DEBUGGING_MEM */ 342 343 344 /* 345 * 346 * Kernel's Virtual Memory Layout. 347 * /-----------------------\ 348 * 0xFFFFFFFF.FFFFFFFF -| |- 349 * | OBP's virtual page | 350 * | tables | 351 * 0xFFFFFFFC.00000000 -|-----------------------|- 352 * : : 353 * : : 354 * -|-----------------------|- 355 * | segzio | (base and size vary) 356 * 0xFFFFFE00.00000000 -|-----------------------|- 357 * | | Ultrasparc I/II support 358 * | segkpm segment | up to 2TB of physical 359 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 360 * | | 361 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 362 * : : 363 * : : 364 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 365 * | | ^ 366 * | UltraSPARC I/II call | | 367 * | bug requires an extra | | 368 * | 4 GB of space between | | 369 * | hole and used RAM | | 370 * | | | 371 * 0xFFFFF800.00000000 -|-----------------------|- | 372 * | | | 373 * | Virtual Address Hole | UltraSPARC 374 * | on UltraSPARC I/II | I/II * ONLY * 375 * | | | 376 * 0x00000800.00000000 -|-----------------------|- | 377 * | | | 378 * | UltraSPARC I/II call | | 379 * | bug requires an extra | | 380 * | 4 GB of space between | | 381 * | hole and used RAM | | 382 * | | v 383 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 384 * : : ^ 385 * : : | 386 * |-----------------------| | 387 * | | | 388 * | ecache flush area | | 389 * | (twice largest e$) | | 390 * | | | 391 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 392 * | overmapped area | alignend_end | 393 * | (kmem64_alignsize | | 394 * | boundary) | | 395 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 396 * | | | 397 * | 64-bit kernel ONLY | | 398 * | | | 399 * | kmem64 segment | | 400 * | | | 401 * | (Relocated extra HME | Approximately 402 * | block allocations, | 1 TB of virtual 403 * | memnode freelists, | address space 404 * | HME hash buckets, | | 405 * | mml_table, kpmp_table,| | 406 * | page_t array and | | 407 * | hashblock pool to | | 408 * | avoid hard-coded | | 409 * | 32-bit vaddr | | 410 * | limitations) | | 411 * | | v 412 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 413 * | | 414 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 415 * | | 416 * 0x00000300.00000000 -|-----------------------|- SYSBASE 417 * : : 418 * : : 419 * -|-----------------------|- 420 * | | 421 * | segmap segment | SEGMAPSIZE (1/8th physmem, 422 * | | 256G MAX) 423 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 424 * : : 425 * : : 426 * -|-----------------------|- 427 * | | 428 * | segkp | SEGKPSIZE (2GB) 429 * | | 430 * | | 431 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 432 * | | 433 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 434 * | | (SEGKPBASE - 0x400000) 435 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 436 * | | (MEMSCRUBBASE - NCARGS) 437 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 438 * | | (ARGSBASE - PPMAPSIZE) 439 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 440 * | | 441 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 442 * | | 443 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 444 * : : 445 * : : 446 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 447 * | | 448 * | OBP | 449 * | | 450 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 451 * | kmdb | 452 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 453 * : : 454 * : : 455 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 456 * | | 457 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 458 * | | ~64MB) 459 * 0x00000000.70002000 -|-----------------------| 460 * | panicbuf | 461 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 462 * | boot-time | 463 * | temporary space | 464 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 465 * : : 466 * : : 467 * | | 468 * |-----------------------|- econtig32 469 * | vm structures | 470 * 0x00000000.01C00000 |-----------------------|- nalloc_end 471 * | TSBs | 472 * |-----------------------|- end/nalloc_base 473 * | kernel data & bss | 474 * 0x00000000.01800000 -|-----------------------| 475 * : nucleus text hole : 476 * 0x00000000.01400000 -|-----------------------| 477 * : : 478 * |-----------------------| 479 * | module text | 480 * |-----------------------|- e_text/modtext 481 * | kernel text | 482 * |-----------------------| 483 * | trap table (48k) | 484 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 485 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 486 * |-----------------------| 487 * | | 488 * | invalid | 489 * | | 490 * 0x00000000.00000000 _|_______________________| 491 * 492 * 493 * 494 * 32-bit User Virtual Memory Layout. 495 * /-----------------------\ 496 * | | 497 * | invalid | 498 * | | 499 * 0xFFC00000 -|-----------------------|- USERLIMIT 500 * | user stack | 501 * : : 502 * : : 503 * : : 504 * | user data | 505 * -|-----------------------|- 506 * | user text | 507 * 0x00002000 -|-----------------------|- 508 * | invalid | 509 * 0x00000000 _|_______________________| 510 * 511 * 512 * 513 * 64-bit User Virtual Memory Layout. 514 * /-----------------------\ 515 * | | 516 * | invalid | 517 * | | 518 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 519 * | user stack | 520 * : : 521 * : : 522 * : : 523 * | user data | 524 * -|-----------------------|- 525 * | user text | 526 * 0x00000000.01000000 -|-----------------------|- 527 * | invalid | 528 * 0x00000000.00000000 _|_______________________| 529 */ 530 531 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 532 extern uint64_t ecache_flush_address(void); 533 534 #pragma weak load_platform_modules 535 #pragma weak plat_startup_memlist 536 #pragma weak ecache_init_scrub_flush_area 537 #pragma weak ecache_flush_address 538 539 540 /* 541 * By default the DR Cage is enabled for maximum OS 542 * MPSS performance. Users needing to disable the cage mechanism 543 * can set this variable to zero via /etc/system. 544 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 545 * will result in loss of DR functionality. 546 * Platforms wishing to disable kernel Cage by default 547 * should do so in their set_platform_defaults() routine. 548 */ 549 int kernel_cage_enable = 1; 550 551 static void 552 setup_cage_params(void) 553 { 554 void (*func)(void); 555 556 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 557 if (func != NULL) { 558 (*func)(); 559 return; 560 } 561 562 if (kernel_cage_enable == 0) { 563 return; 564 } 565 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 566 567 if (kcage_on) { 568 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 569 } else { 570 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 571 } 572 573 } 574 575 /* 576 * Machine-dependent startup code 577 */ 578 void 579 startup(void) 580 { 581 startup_init(); 582 if (&startup_platform) 583 startup_platform(); 584 startup_memlist(); 585 startup_modules(); 586 setup_cage_params(); 587 startup_bop_gone(); 588 startup_vm(); 589 startup_end(); 590 } 591 592 struct regs sync_reg_buf; 593 uint64_t sync_tt; 594 595 void 596 sync_handler(void) 597 { 598 struct panic_trap_info ti; 599 int i; 600 601 /* 602 * Prevent trying to talk to the other CPUs since they are 603 * sitting in the prom and won't reply. 604 */ 605 for (i = 0; i < NCPU; i++) { 606 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 607 cpu[i]->cpu_flags &= ~CPU_READY; 608 cpu[i]->cpu_flags |= CPU_QUIESCED; 609 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 610 } 611 } 612 613 /* 614 * We've managed to get here without going through the 615 * normal panic code path. Try and save some useful 616 * information. 617 */ 618 if (!panicstr && (curthread->t_panic_trap == NULL)) { 619 ti.trap_type = sync_tt; 620 ti.trap_regs = &sync_reg_buf; 621 ti.trap_addr = NULL; 622 ti.trap_mmu_fsr = 0x0; 623 624 curthread->t_panic_trap = &ti; 625 } 626 627 /* 628 * If we're re-entering the panic path, update the signature 629 * block so that the SC knows we're in the second part of panic. 630 */ 631 if (panicstr) 632 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 633 634 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 635 panic("sync initiated"); 636 } 637 638 639 static void 640 startup_init(void) 641 { 642 /* 643 * We want to save the registers while we're still in OBP 644 * so that we know they haven't been fiddled with since. 645 * (In principle, OBP can't change them just because it 646 * makes a callback, but we'd rather not depend on that 647 * behavior.) 648 */ 649 char sync_str[] = 650 "warning @ warning off : sync " 651 "%%tl-c %%tstate h# %p x! " 652 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 653 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 654 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 655 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 656 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 657 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 658 "%%y h# %p l! %%tl-c %%tt h# %p x! " 659 "sync ; warning !"; 660 661 /* 662 * 20 == num of %p substrings 663 * 16 == max num of chars %p will expand to. 664 */ 665 char bp[sizeof (sync_str) + 16 * 20]; 666 667 /* 668 * Initialize ptl1 stack for the 1st CPU. 669 */ 670 ptl1_init_cpu(&cpu0); 671 672 /* 673 * Initialize the address map for cache consistent mappings 674 * to random pages; must be done after vac_size is set. 675 */ 676 ppmapinit(); 677 678 /* 679 * Initialize the PROM callback handler. 680 */ 681 init_vx_handler(); 682 683 /* 684 * have prom call sync_callback() to handle the sync and 685 * save some useful information which will be stored in the 686 * core file later. 687 */ 688 (void) sprintf((char *)bp, sync_str, 689 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 690 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 691 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 692 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 693 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 694 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 695 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 696 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 697 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 698 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 699 prom_interpret(bp, 0, 0, 0, 0, 0); 700 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 701 } 702 703 704 size_t 705 calc_pp_sz(pgcnt_t npages) 706 { 707 708 return (npages * sizeof (struct page)); 709 } 710 711 size_t 712 calc_kpmpp_sz(pgcnt_t npages) 713 { 714 715 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 716 kpm_pgsz = 1ull << kpm_pgshft; 717 kpm_pgoff = kpm_pgsz - 1; 718 kpmp2pshft = kpm_pgshft - PAGESHIFT; 719 kpmpnpgs = 1 << kpmp2pshft; 720 721 if (kpm_smallpages == 0) { 722 /* 723 * Avoid fragmentation problems in kphysm_init() 724 * by allocating for all of physical memory 725 */ 726 kpm_npages = ptokpmpr(physinstalled); 727 return (kpm_npages * sizeof (kpm_page_t)); 728 } else { 729 kpm_npages = npages; 730 return (kpm_npages * sizeof (kpm_spage_t)); 731 } 732 } 733 734 size_t 735 calc_pagehash_sz(pgcnt_t npages) 736 { 737 738 /* 739 * The page structure hash table size is a power of 2 740 * such that the average hash chain length is PAGE_HASHAVELEN. 741 */ 742 page_hashsz = npages / PAGE_HASHAVELEN; 743 page_hashsz = 1 << highbit(page_hashsz); 744 return (page_hashsz * sizeof (struct page *)); 745 } 746 747 int testkmem64_smchunks = 0; 748 749 int 750 alloc_kmem64(caddr_t base, caddr_t end) 751 { 752 int i; 753 caddr_t aligned_end = NULL; 754 755 if (testkmem64_smchunks) 756 return (1); 757 758 /* 759 * Make one large memory alloc after figuring out the 64-bit size. This 760 * will enable use of the largest page size appropriate for the system 761 * architecture. 762 */ 763 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 764 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 765 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 766 size_t alloc_size, alignsize; 767 #if !defined(C_OBP) 768 unsigned long long pa; 769 #endif /* !C_OBP */ 770 771 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 772 continue; 773 alignsize = TTEBYTES(i); 774 kmem64_szc = i; 775 776 /* limit page size for small memory */ 777 if (mmu_btop(alignsize) > (npages >> 2)) 778 continue; 779 780 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 781 alloc_size = aligned_end - base; 782 #if !defined(C_OBP) 783 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 784 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 785 kmem64_pabase = pa; 786 kmem64_aligned_end = aligned_end; 787 install_kmem64_tte(); 788 break; 789 } else { 790 prom_free_phys(alloc_size, pa); 791 } 792 } 793 #else /* !C_OBP */ 794 if (prom_alloc(base, alloc_size, alignsize) == base) { 795 kmem64_pabase = va_to_pa(kmem64_base); 796 kmem64_aligned_end = aligned_end; 797 break; 798 } 799 #endif /* !C_OBP */ 800 if (i == TTE8K) { 801 #ifdef sun4v 802 /* return failure to try small allocations */ 803 return (1); 804 #else 805 prom_panic("kmem64 allocation failure"); 806 #endif 807 } 808 } 809 ASSERT(aligned_end != NULL); 810 return (0); 811 } 812 813 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 814 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 815 816 #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 817 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 818 (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 819 820 #if !defined(C_OBP) 821 /* 822 * Install a temporary tte handler in OBP for kmem64 area. 823 * 824 * We map kmem64 area with large pages before the trap table is taken 825 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 826 * the same area. Duplicate tlb entries with different page sizes 827 * cause unpredicatble behavior. To avoid this, we don't create 828 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 829 * OBP). Instead, we manage translations with a temporary va>tte-data 830 * handler (kmem64-tte). This handler is replaced by unix-tte when 831 * the trap table is taken over. 832 * 833 * The temporary handler knows the physical address of the kmem64 834 * area. It uses the prom's pgmap@ Forth word for other addresses. 835 * 836 * We have to use BOP_ALLOC() method for C-OBP platforms because 837 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 838 * sun4u platforms. On sun4u we flush tlb after trap table is taken 839 * over if we use large pages for kernel heap and kmem64. Since sun4u 840 * prom (unlike sun4v) calls va>tte-data first for client address 841 * translation prom's ttes for kmem64 can't get into TLB even if we 842 * later switch to prom's trap table again. C-OBP uses 4M pages for 843 * client mappings when possible so on all platforms we get the 844 * benefit from large mappings for kmem64 area immediately during 845 * boot. 846 * 847 * pseudo code: 848 * if (context != 0) { 849 * return false 850 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 851 * tte = tte_template + 852 * (((miss_va & pagemask) - kmem64_base)); 853 * return tte, true 854 * } else { 855 * return pgmap@ result 856 * } 857 */ 858 char kmem64_obp_str[] = 859 "h# %lx constant kmem64-base " 860 "h# %lx constant kmem64-end " 861 "h# %lx constant kmem64-pagemask " 862 "h# %lx constant kmem64-template " 863 864 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 865 " if ( addr ) " 866 " drop false exit then ( false ) " 867 " dup kmem64-base kmem64-end within if ( addr ) " 868 " kmem64-pagemask and ( addr' ) " 869 " kmem64-base - ( addr' ) " 870 " kmem64-template + ( tte ) " 871 " true ( tte true ) " 872 " else ( addr ) " 873 " pgmap@ ( tte ) " 874 " dup 0< if true else drop false then ( tte true | false ) " 875 " then ( tte true | false ) " 876 "; " 877 878 "' kmem64-tte is va>tte-data " 879 ; 880 881 static void 882 install_kmem64_tte() 883 { 884 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 885 tte_t tte; 886 887 PRM_DEBUG(kmem64_pabase); 888 PRM_DEBUG(kmem64_szc); 889 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 890 PROC_DATA | HAT_NOSYNC, kmem64_szc); 891 PRM_DEBUG(tte.ll); 892 (void) sprintf(b, kmem64_obp_str, 893 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 894 ASSERT(strlen(b) < sizeof (b)); 895 prom_interpret(b, 0, 0, 0, 0, 0); 896 } 897 #endif /* !C_OBP */ 898 899 /* 900 * As OBP takes up some RAM when the system boots, pages will already be "lost" 901 * to the system and reflected in npages by the time we see it. 902 * 903 * We only want to allocate kernel structures in the 64-bit virtual address 904 * space on systems with enough RAM to make the overhead of keeping track of 905 * an extra kernel memory segment worthwhile. 906 * 907 * Since OBP has already performed its memory allocations by this point, if we 908 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 909 * memory in the 64-bit virtual address space; otherwise keep allocations 910 * contiguous with we've mapped so far in the 32-bit virtual address space. 911 */ 912 #define MINMOVE_RAM_MB ((size_t)1900) 913 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 914 #define BYTES_TO_MB(b) ((b) / 1048576ul) 915 916 pgcnt_t tune_npages = (pgcnt_t) 917 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 918 919 #pragma weak page_set_colorequiv_arr_cpu 920 extern void page_set_colorequiv_arr_cpu(void); 921 extern void page_set_colorequiv_arr(void); 922 923 static pgcnt_t ramdisk_npages; 924 static struct memlist *old_phys_avail; 925 926 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 927 928 static void 929 startup_memlist(void) 930 { 931 size_t hmehash_sz, pagelist_sz, tt_sz; 932 size_t psetable_sz; 933 caddr_t alloc_base; 934 caddr_t memspace; 935 struct memlist *cur; 936 size_t syslimit = (size_t)SYSLIMIT; 937 size_t sysbase = (size_t)SYSBASE; 938 939 /* 940 * Initialize enough of the system to allow kmem_alloc to work by 941 * calling boot to allocate its memory until the time that 942 * kvm_init is completed. The page structs are allocated after 943 * rounding up end to the nearest page boundary; the memsegs are 944 * initialized and the space they use comes from the kernel heap. 945 * With appropriate initialization, they can be reallocated later 946 * to a size appropriate for the machine's configuration. 947 * 948 * At this point, memory is allocated for things that will never 949 * need to be freed, this used to be "valloced". This allows a 950 * savings as the pages don't need page structures to describe 951 * them because them will not be managed by the vm system. 952 */ 953 954 /* 955 * We're loaded by boot with the following configuration (as 956 * specified in the sun4u/conf/Mapfile): 957 * 958 * text: 4 MB chunk aligned on a 4MB boundary 959 * data & bss: 4 MB chunk aligned on a 4MB boundary 960 * 961 * These two chunks will eventually be mapped by 2 locked 4MB 962 * ttes and will represent the nucleus of the kernel. This gives 963 * us some free space that is already allocated, some or all of 964 * which is made available to kernel module text. 965 * 966 * The free space in the data-bss chunk is used for nucleus 967 * allocatable data structures and we reserve it using the 968 * nalloc_base and nalloc_end variables. This space is currently 969 * being used for hat data structures required for tlb miss 970 * handling operations. We align nalloc_base to a l2 cache 971 * linesize because this is the line size the hardware uses to 972 * maintain cache coherency. 973 * 512K is carved out for module data. 974 */ 975 976 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 977 e_moddata = moddata + MODDATA; 978 nalloc_base = e_moddata; 979 980 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 981 valloc_base = nalloc_base; 982 983 /* 984 * Calculate the start of the data segment. 985 */ 986 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 987 prom_panic("nucleus data overflow"); 988 989 PRM_DEBUG(moddata); 990 PRM_DEBUG(nalloc_base); 991 PRM_DEBUG(nalloc_end); 992 993 /* 994 * Remember any slop after e_text so we can give it to the modules. 995 */ 996 PRM_DEBUG(e_text); 997 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 998 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 999 prom_panic("nucleus text overflow"); 1000 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1001 modtext; 1002 PRM_DEBUG(modtext); 1003 PRM_DEBUG(modtext_sz); 1004 1005 init_boot_memlists(); 1006 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1007 &boot_physavail, &boot_physavail_len, 1008 &boot_virtavail, &boot_virtavail_len); 1009 1010 /* 1011 * Remember what the physically available highest page is 1012 * so that dumpsys works properly, and find out how much 1013 * memory is installed. 1014 */ 1015 installed_top_size_memlist_array(boot_physinstalled, 1016 boot_physinstalled_len, &physmax, &physinstalled); 1017 PRM_DEBUG(physinstalled); 1018 PRM_DEBUG(physmax); 1019 1020 /* Fill out memory nodes config structure */ 1021 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1022 1023 /* 1024 * npages is the maximum of available physical memory possible. 1025 * (ie. it will never be more than this) 1026 * 1027 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1028 * using phys_avail will underestimate what will end up being freed. 1029 * A better initial guess is just total memory minus the kernel text 1030 */ 1031 npages = physinstalled - btop(MMU_PAGESIZE4M); 1032 1033 /* 1034 * First allocate things that can go in the nucleus data page 1035 * (fault status, TSBs, dmv, CPUs) 1036 */ 1037 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1038 1039 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1040 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1041 1042 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1043 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1044 1045 if (ndata_alloc_dmv(&ndata) != 0) 1046 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1047 1048 if (ndata_alloc_page_mutexs(&ndata) != 0) 1049 cmn_err(CE_PANIC, 1050 "no more nucleus memory after page free lists alloc"); 1051 1052 if (ndata_alloc_hat(&ndata, npages) != 0) 1053 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1054 1055 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1056 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1057 1058 /* 1059 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1060 * 1061 * There are comments all over the SFMMU code warning of dire 1062 * consequences if the TSBs are moved out of 32-bit space. This 1063 * is largely because the asm code uses "sethi %hi(addr)"-type 1064 * instructions which will not provide the expected result if the 1065 * address is a 64-bit one. 1066 * 1067 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1068 */ 1069 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1070 PRM_DEBUG(alloc_base); 1071 1072 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1073 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1074 PRM_DEBUG(alloc_base); 1075 1076 /* 1077 * Allocate IOMMU TSB array. We do this here so that the physical 1078 * memory gets deducted from the PROM's physical memory list. 1079 */ 1080 alloc_base = iommu_tsb_init(alloc_base); 1081 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1082 PRM_DEBUG(alloc_base); 1083 1084 /* 1085 * Allow for an early allocation of physically contiguous memory. 1086 */ 1087 alloc_base = contig_mem_prealloc(alloc_base, npages); 1088 1089 /* 1090 * Platforms like Starcat and OPL need special structures assigned in 1091 * 32-bit virtual address space because their probing routines execute 1092 * FCode, and FCode can't handle 64-bit virtual addresses... 1093 */ 1094 if (&plat_startup_memlist) { 1095 alloc_base = plat_startup_memlist(alloc_base); 1096 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1097 ecache_alignsize); 1098 PRM_DEBUG(alloc_base); 1099 } 1100 1101 /* 1102 * Save off where the contiguous allocations to date have ended 1103 * in econtig32. 1104 */ 1105 econtig32 = alloc_base; 1106 PRM_DEBUG(econtig32); 1107 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1108 cmn_err(CE_PANIC, "econtig32 too big"); 1109 1110 pp_sz = calc_pp_sz(npages); 1111 PRM_DEBUG(pp_sz); 1112 if (kpm_enable) { 1113 kpm_pp_sz = calc_kpmpp_sz(npages); 1114 PRM_DEBUG(kpm_pp_sz); 1115 } 1116 1117 hmehash_sz = calc_hmehash_sz(npages); 1118 PRM_DEBUG(hmehash_sz); 1119 1120 pagehash_sz = calc_pagehash_sz(npages); 1121 PRM_DEBUG(pagehash_sz); 1122 1123 pagelist_sz = calc_free_pagelist_sz(); 1124 PRM_DEBUG(pagelist_sz); 1125 1126 #ifdef TRAPTRACE 1127 tt_sz = calc_traptrace_sz(); 1128 PRM_DEBUG(tt_sz); 1129 #else 1130 tt_sz = 0; 1131 #endif /* TRAPTRACE */ 1132 1133 /* 1134 * Place the array that protects pp->p_selock in the kmem64 wad. 1135 */ 1136 pse_shift = size_pse_array(npages, max_ncpus); 1137 PRM_DEBUG(pse_shift); 1138 pse_table_size = 1 << pse_shift; 1139 PRM_DEBUG(pse_table_size); 1140 psetable_sz = roundup( 1141 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1142 PRM_DEBUG(psetable_sz); 1143 1144 /* 1145 * Now allocate the whole wad 1146 */ 1147 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1148 pagelist_sz + tt_sz + psetable_sz; 1149 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1150 kmem64_base = (caddr_t)syslimit; 1151 kmem64_end = kmem64_base + kmem64_sz; 1152 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1153 /* 1154 * Attempt for kmem64 to allocate one big 1155 * contiguous chunk of memory failed. 1156 * We get here because we are sun4v. 1157 * We will proceed by breaking up 1158 * the allocation into two attempts. 1159 * First, we allocate kpm_pp_sz, hmehash_sz, 1160 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1161 * one contiguous chunk. This is a much smaller 1162 * chunk and we should get it, if not we panic. 1163 * Note that hmehash and tt need to be physically 1164 * (in the real address sense) contiguous. 1165 * Next, we use bop_alloc_chunk() to 1166 * to allocate the page_t structures. 1167 * This will allow the page_t to be allocated 1168 * in multiple smaller chunks. 1169 * In doing so, the assumption that page_t is 1170 * physically contiguous no longer hold, this is ok 1171 * for sun4v but not for sun4u. 1172 */ 1173 size_t tmp_size; 1174 caddr_t tmp_base; 1175 1176 pp_sz = roundup(pp_sz, PAGESIZE); 1177 1178 /* 1179 * Allocate kpm_pp_sz, hmehash_sz, 1180 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1181 */ 1182 tmp_base = kmem64_base + pp_sz; 1183 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1184 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1185 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1186 prom_panic("kmem64 prom_alloc contig failed"); 1187 PRM_DEBUG(tmp_base); 1188 PRM_DEBUG(tmp_size); 1189 1190 /* 1191 * Allocate the page_ts 1192 */ 1193 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1194 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1195 PRM_DEBUG(kmem64_base); 1196 PRM_DEBUG(pp_sz); 1197 1198 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1199 ASSERT(kmem64_aligned_end >= kmem64_end); 1200 1201 kmem64_smchunks = 1; 1202 } else { 1203 1204 /* 1205 * We need to adjust pp_sz for the normal 1206 * case where kmem64 can allocate one large chunk 1207 */ 1208 if (kpm_smallpages == 0) { 1209 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1210 } else { 1211 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1212 sizeof (kpm_spage_t)); 1213 } 1214 pp_sz = npages * sizeof (struct page); 1215 } 1216 1217 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1218 cmn_err(CE_PANIC, "not enough kmem64 space"); 1219 PRM_DEBUG(kmem64_base); 1220 PRM_DEBUG(kmem64_end); 1221 PRM_DEBUG(kmem64_aligned_end); 1222 1223 /* 1224 * ... and divy it up 1225 */ 1226 alloc_base = kmem64_base; 1227 1228 pp_base = (page_t *)alloc_base; 1229 alloc_base += pp_sz; 1230 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1231 PRM_DEBUG(pp_base); 1232 PRM_DEBUG(npages); 1233 1234 if (kpm_enable) { 1235 kpm_pp_base = alloc_base; 1236 if (kpm_smallpages == 0) { 1237 /* kpm_npages based on physinstalled, don't reset */ 1238 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1239 } else { 1240 kpm_npages = ptokpmpr(npages); 1241 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1242 } 1243 alloc_base += kpm_pp_sz; 1244 alloc_base = 1245 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1246 PRM_DEBUG(kpm_pp_base); 1247 } 1248 1249 alloc_base = alloc_hmehash(alloc_base); 1250 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1251 PRM_DEBUG(alloc_base); 1252 1253 page_hash = (page_t **)alloc_base; 1254 alloc_base += pagehash_sz; 1255 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1256 PRM_DEBUG(page_hash); 1257 1258 alloc_base = alloc_page_freelists(alloc_base); 1259 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1260 PRM_DEBUG(alloc_base); 1261 1262 #ifdef TRAPTRACE 1263 ttrace_buf = alloc_base; 1264 alloc_base += tt_sz; 1265 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1266 PRM_DEBUG(alloc_base); 1267 #endif /* TRAPTRACE */ 1268 1269 pse_mutex = (pad_mutex_t *)alloc_base; 1270 alloc_base += psetable_sz; 1271 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1272 PRM_DEBUG(alloc_base); 1273 1274 /* 1275 * Note that if we use small chunk allocations for 1276 * kmem64, we need to ensure kmem64_end is the same as 1277 * kmem64_aligned_end to prevent subsequent logic from 1278 * trying to reuse the overmapping. 1279 * Otherwise we adjust kmem64_end to what we really allocated. 1280 */ 1281 if (kmem64_smchunks) { 1282 kmem64_end = kmem64_aligned_end; 1283 } else { 1284 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1285 } 1286 kmem64_sz = kmem64_end - kmem64_base; 1287 1288 if (&ecache_init_scrub_flush_area) { 1289 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1290 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1291 } 1292 1293 /* 1294 * If physmem is patched to be non-zero, use it instead of 1295 * the monitor value unless physmem is larger than the total 1296 * amount of memory on hand. 1297 */ 1298 if (physmem == 0 || physmem > npages) 1299 physmem = npages; 1300 1301 /* 1302 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1303 * is mounted as root. This memory is held down by OBP and unlike 1304 * the stub boot_archive is never released. 1305 * 1306 * In order to get things sized correctly on lower memory 1307 * machines (where the memory used by the ramdisk represents 1308 * a significant portion of memory), physmem is adjusted. 1309 * 1310 * This is done by subtracting the ramdisk_size which is set 1311 * to the size of the ramdisk (in Kb) in /etc/system at the 1312 * time the miniroot archive is constructed. 1313 */ 1314 if (root_is_ramdisk == B_TRUE) { 1315 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1316 physmem -= ramdisk_npages; 1317 } 1318 1319 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1320 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1321 1322 /* 1323 * Allocate space for the interrupt vector table. 1324 */ 1325 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1326 if (memspace != (caddr_t)intr_vec_table) 1327 prom_panic("interrupt vector table allocation failure"); 1328 1329 /* 1330 * Between now and when we finish copying in the memory lists, 1331 * allocations happen so the space gets fragmented and the 1332 * lists longer. Leave enough space for lists twice as 1333 * long as we have now; then roundup to a pagesize. 1334 */ 1335 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1336 prom_phys_avail_len() + prom_virt_avail_len()); 1337 memlist_sz *= 2; 1338 memlist_sz = roundup(memlist_sz, PAGESIZE); 1339 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1340 if (memspace == NULL) 1341 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1342 1343 memlist = (struct memlist *)memspace; 1344 memlist_end = (char *)memspace + memlist_sz; 1345 PRM_DEBUG(memlist); 1346 PRM_DEBUG(memlist_end); 1347 1348 PRM_DEBUG(sysbase); 1349 PRM_DEBUG(syslimit); 1350 kernelheap_init((void *)sysbase, (void *)syslimit, 1351 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1352 1353 /* 1354 * Take the most current snapshot we can by calling mem-update. 1355 */ 1356 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1357 &boot_physavail, &boot_physavail_len, 1358 &boot_virtavail, &boot_virtavail_len); 1359 1360 /* 1361 * Remove the space used by prom_alloc from the kernel heap 1362 * plus the area actually used by the OBP (if any) 1363 * ignoring virtual addresses in virt_avail, above syslimit. 1364 */ 1365 virt_avail = memlist; 1366 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1367 1368 for (cur = virt_avail; cur->next; cur = cur->next) { 1369 uint64_t range_base, range_size; 1370 1371 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1372 continue; 1373 if (range_base >= (uint64_t)syslimit) 1374 break; 1375 /* 1376 * Limit the range to end at syslimit. 1377 */ 1378 range_size = MIN(cur->next->address, 1379 (uint64_t)syslimit) - range_base; 1380 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1381 0, 0, (void *)range_base, (void *)(range_base + range_size), 1382 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1383 } 1384 1385 phys_avail = memlist; 1386 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1387 1388 /* 1389 * Add any extra memory at the end of the ndata region if there's at 1390 * least a page to add. There might be a few more pages available in 1391 * the middle of the ndata region, but for now they are ignored. 1392 */ 1393 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1394 if (nalloc_base == NULL) 1395 nalloc_base = nalloc_end; 1396 ndata_remain_sz = nalloc_end - nalloc_base; 1397 1398 /* 1399 * Copy physinstalled list into kernel space. 1400 */ 1401 phys_install = memlist; 1402 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1403 1404 /* 1405 * Create list of physical addrs we don't need pp's for: 1406 * kernel text 4M page 1407 * kernel data 4M page - ndata_remain_sz 1408 * kmem64 pages 1409 * 1410 * NB if adding any pages here, make sure no kpm page 1411 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1412 */ 1413 nopp_list = memlist; 1414 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1415 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1416 &memlist, &nopp_list); 1417 1418 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1419 if (!kmem64_smchunks) 1420 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1421 1422 if ((caddr_t)memlist > (memspace + memlist_sz)) 1423 prom_panic("memlist overflow"); 1424 1425 /* 1426 * Size the pcf array based on the number of cpus in the box at 1427 * boot time. 1428 */ 1429 pcf_init(); 1430 1431 /* 1432 * Initialize the page structures from the memory lists. 1433 */ 1434 kphysm_init(); 1435 1436 availrmem_initial = availrmem = freemem; 1437 PRM_DEBUG(availrmem); 1438 1439 /* 1440 * Some of the locks depend on page_hashsz being set! 1441 * kmem_init() depends on this; so, keep it here. 1442 */ 1443 page_lock_init(); 1444 1445 /* 1446 * Initialize kernel memory allocator. 1447 */ 1448 kmem_init(); 1449 1450 /* 1451 * Factor in colorequiv to check additional 'equivalent' bins 1452 */ 1453 if (&page_set_colorequiv_arr_cpu != NULL) 1454 page_set_colorequiv_arr_cpu(); 1455 else 1456 page_set_colorequiv_arr(); 1457 1458 /* 1459 * Initialize bp_mapin(). 1460 */ 1461 bp_init(shm_alignment, HAT_STRICTORDER); 1462 1463 /* 1464 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1465 * vector data structures and MPO mblock structs from the 32-bit heap. 1466 */ 1467 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1468 panicbuf, panicbuf + PANICBUFSIZE, 1469 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1470 1471 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1472 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1473 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1474 1475 if (mpo_heap32_bufsz > (size_t)0) { 1476 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1477 PAGESIZE, 0, 0, mpo_heap32_buf, 1478 mpo_heap32_buf + mpo_heap32_bufsz, 1479 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1480 } 1481 mem_config_init(); 1482 } 1483 1484 static void 1485 startup_modules(void) 1486 { 1487 int nhblk1, nhblk8; 1488 size_t nhblksz; 1489 pgcnt_t pages_per_hblk; 1490 size_t hme8blk_sz, hme1blk_sz; 1491 1492 /* 1493 * Let the platforms have a chance to change default 1494 * values before reading system file. 1495 */ 1496 if (&set_platform_defaults) 1497 set_platform_defaults(); 1498 1499 /* 1500 * Calculate default settings of system parameters based upon 1501 * maxusers, yet allow to be overridden via the /etc/system file. 1502 */ 1503 param_calc(0); 1504 1505 mod_setup(); 1506 1507 /* 1508 * If this is a positron, complain and halt. 1509 */ 1510 if (&iam_positron && iam_positron()) { 1511 cmn_err(CE_WARN, "This hardware platform is not supported" 1512 " by this release of Solaris.\n"); 1513 #ifdef DEBUG 1514 prom_enter_mon(); /* Type 'go' to resume */ 1515 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1516 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1517 1518 #else /* DEBUG */ 1519 halt(0); 1520 #endif /* DEBUG */ 1521 } 1522 1523 /* 1524 * If we are running firmware that isn't 64-bit ready 1525 * then complain and halt. 1526 */ 1527 do_prom_version_check(); 1528 1529 /* 1530 * Initialize system parameters 1531 */ 1532 param_init(); 1533 1534 /* 1535 * maxmem is the amount of physical memory we're playing with. 1536 */ 1537 maxmem = physmem; 1538 1539 /* Set segkp limits. */ 1540 ncbase = kdi_segdebugbase; 1541 ncend = kdi_segdebugbase; 1542 1543 /* 1544 * Initialize the hat layer. 1545 */ 1546 hat_init(); 1547 1548 /* 1549 * Initialize segment management stuff. 1550 */ 1551 seg_init(); 1552 1553 /* 1554 * Create the va>tte handler, so the prom can understand 1555 * kernel translations. The handler is installed later, just 1556 * as we are about to take over the trap table from the prom. 1557 */ 1558 create_va_to_tte(); 1559 1560 /* 1561 * Load the forthdebugger (optional) 1562 */ 1563 forthdebug_init(); 1564 1565 /* 1566 * Create OBP node for console input callbacks 1567 * if it is needed. 1568 */ 1569 startup_create_io_node(); 1570 1571 if (modloadonly("fs", "specfs") == -1) 1572 halt("Can't load specfs"); 1573 1574 if (modloadonly("fs", "devfs") == -1) 1575 halt("Can't load devfs"); 1576 1577 if (modloadonly("misc", "swapgeneric") == -1) 1578 halt("Can't load swapgeneric"); 1579 1580 (void) modloadonly("sys", "lbl_edition"); 1581 1582 dispinit(); 1583 1584 /* 1585 * Infer meanings to the members of the idprom buffer. 1586 */ 1587 parse_idprom(); 1588 1589 /* Read cluster configuration data. */ 1590 clconf_init(); 1591 1592 setup_ddi(); 1593 1594 /* 1595 * Lets take this opportunity to load the root device. 1596 */ 1597 if (loadrootmodules() != 0) 1598 debug_enter("Can't load the root filesystem"); 1599 1600 /* 1601 * Load tod driver module for the tod part found on this system. 1602 * Recompute the cpu frequency/delays based on tod as tod part 1603 * tends to keep time more accurately. 1604 */ 1605 if (&load_tod_module) 1606 load_tod_module(); 1607 1608 /* 1609 * Allow platforms to load modules which might 1610 * be needed after bootops are gone. 1611 */ 1612 if (&load_platform_modules) 1613 load_platform_modules(); 1614 1615 setcpudelay(); 1616 1617 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1618 &boot_physavail, &boot_physavail_len, 1619 &boot_virtavail, &boot_virtavail_len); 1620 1621 /* 1622 * Calculation and allocation of hmeblks needed to remap 1623 * the memory allocated by PROM till now. 1624 * Overestimate the number of hblk1 elements by assuming 1625 * worst case of TTE64K mappings. 1626 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1627 */ 1628 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1629 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1630 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1631 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1632 1633 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1634 1635 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1636 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1637 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1638 1639 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1640 1641 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1642 nhblk8 = 0; 1643 while (bop_alloc_pages > 1) { 1644 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1645 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1646 bop_alloc_pages *= hme8blk_sz; 1647 bop_alloc_pages = btopr(bop_alloc_pages); 1648 } 1649 nhblk8 += 2; 1650 1651 /* 1652 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1653 * boundary, the number of hblk8's needed to map the entries in the 1654 * boot_virtavail list needs to be adjusted to take this into 1655 * consideration. Thus, we need to add additional hblk8's since it 1656 * is possible that an hblk8 will not have all 8 slots used due to 1657 * alignment constraints. Since there were boot_virtavail_len entries 1658 * in that list, we need to add that many hblk8's to the number 1659 * already calculated to make sure we don't underestimate. 1660 */ 1661 nhblk8 += boot_virtavail_len; 1662 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1663 1664 /* Allocate in pagesize chunks */ 1665 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1666 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1667 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1668 } 1669 1670 static void 1671 startup_bop_gone(void) 1672 { 1673 1674 /* 1675 * Destroy the MD initialized at startup 1676 * The startup initializes the MD framework 1677 * using prom and BOP alloc free it now. 1678 */ 1679 mach_descrip_startup_fini(); 1680 1681 /* 1682 * We're done with prom allocations. 1683 */ 1684 bop_fini(); 1685 1686 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1687 &boot_physavail, &boot_physavail_len, 1688 &boot_virtavail, &boot_virtavail_len); 1689 1690 /* 1691 * setup physically contiguous area twice as large as the ecache. 1692 * this is used while doing displacement flush of ecaches 1693 */ 1694 if (&ecache_flush_address) { 1695 ecache_flushaddr = ecache_flush_address(); 1696 if (ecache_flushaddr == (uint64_t)-1) { 1697 cmn_err(CE_PANIC, 1698 "startup: no memory to set ecache_flushaddr"); 1699 } 1700 } 1701 1702 /* 1703 * Virtual available next. 1704 */ 1705 ASSERT(virt_avail != NULL); 1706 memlist_free_list(virt_avail); 1707 virt_avail = memlist; 1708 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1709 1710 } 1711 1712 1713 /* 1714 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1715 * allocations have been performed. We can't call it in startup_bop_gone 1716 * since later operations can cause obp to allocate more memory. 1717 */ 1718 void 1719 startup_fixup_physavail(void) 1720 { 1721 struct memlist *cur; 1722 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1723 1724 PRM_DEBUG(kmem64_overmap_size); 1725 1726 /* 1727 * take the most current snapshot we can by calling mem-update 1728 */ 1729 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1730 &boot_physavail, &boot_physavail_len, 1731 &boot_virtavail, &boot_virtavail_len); 1732 1733 /* 1734 * Copy phys_avail list, again. 1735 * Both the kernel/boot and the prom have been allocating 1736 * from the original list we copied earlier. 1737 */ 1738 cur = memlist; 1739 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1740 1741 /* 1742 * Add any unused kmem64 memory from overmapped page 1743 * (Note: va_to_pa does not work for kmem64_end) 1744 */ 1745 if (kmem64_overmap_size) { 1746 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1747 kmem64_overmap_size, &memlist, &cur); 1748 } 1749 1750 /* 1751 * Add any extra memory after e_data we added to the phys_avail list 1752 * back to the old list. 1753 */ 1754 if (ndata_remain_sz >= MMU_PAGESIZE) 1755 memlist_add(va_to_pa(nalloc_base), 1756 (uint64_t)ndata_remain_sz, &memlist, &cur); 1757 1758 /* 1759 * There isn't any bounds checking on the memlist area 1760 * so ensure it hasn't overgrown. 1761 */ 1762 if ((caddr_t)memlist > (caddr_t)memlist_end) 1763 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1764 1765 /* 1766 * The kernel removes the pages that were allocated for it from 1767 * the freelist, but we now have to find any -extra- pages that 1768 * the prom has allocated for it's own book-keeping, and remove 1769 * them from the freelist too. sigh. 1770 */ 1771 sync_memlists(phys_avail, cur); 1772 1773 ASSERT(phys_avail != NULL); 1774 1775 old_phys_avail = phys_avail; 1776 phys_avail = cur; 1777 } 1778 1779 void 1780 update_kcage_ranges(uint64_t addr, uint64_t len) 1781 { 1782 pfn_t base = btop(addr); 1783 pgcnt_t num = btop(len); 1784 int rv; 1785 1786 rv = kcage_range_add(base, num, kcage_startup_dir); 1787 1788 if (rv == ENOMEM) { 1789 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1790 (len == 0 ? 0 : BYTES_TO_MB(len))); 1791 } else if (rv != 0) { 1792 /* catch this in debug kernels */ 1793 ASSERT(0); 1794 1795 cmn_err(CE_WARN, "unexpected kcage_range_add" 1796 " return value %d", rv); 1797 } 1798 } 1799 1800 static void 1801 startup_vm(void) 1802 { 1803 size_t i; 1804 struct segmap_crargs a; 1805 struct segkpm_crargs b; 1806 1807 uint64_t avmem; 1808 caddr_t va; 1809 pgcnt_t max_phys_segkp; 1810 int mnode; 1811 1812 extern int use_brk_lpg, use_stk_lpg; 1813 1814 /* 1815 * get prom's mappings, create hments for them and switch 1816 * to the kernel context. 1817 */ 1818 hat_kern_setup(); 1819 1820 /* 1821 * Take over trap table 1822 */ 1823 setup_trap_table(); 1824 1825 /* 1826 * Install the va>tte handler, so that the prom can handle 1827 * misses and understand the kernel table layout in case 1828 * we need call into the prom. 1829 */ 1830 install_va_to_tte(); 1831 1832 /* 1833 * Set a flag to indicate that the tba has been taken over. 1834 */ 1835 tba_taken_over = 1; 1836 1837 /* initialize MMU primary context register */ 1838 mmu_init_kcontext(); 1839 1840 /* 1841 * The boot cpu can now take interrupts, x-calls, x-traps 1842 */ 1843 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1844 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1845 1846 /* 1847 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1848 */ 1849 tbr_wr_addr_inited = 1; 1850 1851 /* 1852 * Initialize VM system, and map kernel address space. 1853 */ 1854 kvm_init(); 1855 1856 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1857 if (kernel_cage_enable) { 1858 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1859 } 1860 memlist_free_list(old_phys_avail); 1861 1862 /* 1863 * If the following is true, someone has patched 1864 * phsymem to be less than the number of pages that 1865 * the system actually has. Remove pages until system 1866 * memory is limited to the requested amount. Since we 1867 * have allocated page structures for all pages, we 1868 * correct the amount of memory we want to remove 1869 * by the size of the memory used to hold page structures 1870 * for the non-used pages. 1871 */ 1872 if (physmem + ramdisk_npages < npages) { 1873 pgcnt_t diff, off; 1874 struct page *pp; 1875 struct seg kseg; 1876 1877 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1878 1879 off = 0; 1880 diff = npages - (physmem + ramdisk_npages); 1881 diff -= mmu_btopr(diff * sizeof (struct page)); 1882 kseg.s_as = &kas; 1883 while (diff--) { 1884 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1885 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1886 &kseg, (caddr_t)off); 1887 if (pp == NULL) 1888 cmn_err(CE_PANIC, "limited physmem too much!"); 1889 page_io_unlock(pp); 1890 page_downgrade(pp); 1891 availrmem--; 1892 off += MMU_PAGESIZE; 1893 } 1894 } 1895 1896 /* 1897 * When printing memory, show the total as physmem less 1898 * that stolen by a debugger. 1899 */ 1900 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1901 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1902 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1903 1904 avmem = (uint64_t)freemem << PAGESHIFT; 1905 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1906 1907 /* 1908 * For small memory systems disable automatic large pages. 1909 */ 1910 if (physmem < privm_lpg_min_physmem) { 1911 use_brk_lpg = 0; 1912 use_stk_lpg = 0; 1913 } 1914 1915 /* 1916 * Perform platform specific freelist processing 1917 */ 1918 if (&plat_freelist_process) { 1919 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1920 if (mem_node_config[mnode].exists) 1921 plat_freelist_process(mnode); 1922 } 1923 1924 /* 1925 * Initialize the segkp segment type. We position it 1926 * after the configured tables and buffers (whose end 1927 * is given by econtig) and before V_WKBASE_ADDR. 1928 * Also in this area is segkmap (size SEGMAPSIZE). 1929 */ 1930 1931 /* XXX - cache alignment? */ 1932 va = (caddr_t)SEGKPBASE; 1933 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1934 1935 max_phys_segkp = (physmem * 2); 1936 1937 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1938 segkpsize = btop(SEGKPDEFSIZE); 1939 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1940 "segkpsize has been reset to %ld pages", segkpsize); 1941 } 1942 1943 i = ptob(MIN(segkpsize, max_phys_segkp)); 1944 1945 rw_enter(&kas.a_lock, RW_WRITER); 1946 if (seg_attach(&kas, va, i, segkp) < 0) 1947 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1948 if (segkp_create(segkp) != 0) 1949 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1950 rw_exit(&kas.a_lock); 1951 1952 /* 1953 * kpm segment 1954 */ 1955 segmap_kpm = kpm_enable && 1956 segmap_kpm && PAGESIZE == MAXBSIZE; 1957 1958 if (kpm_enable) { 1959 rw_enter(&kas.a_lock, RW_WRITER); 1960 1961 /* 1962 * The segkpm virtual range range is larger than the 1963 * actual physical memory size and also covers gaps in 1964 * the physical address range for the following reasons: 1965 * . keep conversion between segkpm and physical addresses 1966 * simple, cheap and unambiguous. 1967 * . avoid extension/shrink of the the segkpm in case of DR. 1968 * . avoid complexity for handling of virtual addressed 1969 * caches, segkpm and the regular mapping scheme must be 1970 * kept in sync wrt. the virtual color of mapped pages. 1971 * Any accesses to virtual segkpm ranges not backed by 1972 * physical memory will fall through the memseg pfn hash 1973 * and will be handled in segkpm_fault. 1974 * Additional kpm_size spaces needed for vac alias prevention. 1975 */ 1976 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1977 segkpm) < 0) 1978 cmn_err(CE_PANIC, "cannot attach segkpm"); 1979 1980 b.prot = PROT_READ | PROT_WRITE; 1981 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1982 1983 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1984 panic("segkpm_create segkpm"); 1985 1986 rw_exit(&kas.a_lock); 1987 1988 mach_kpm_init(); 1989 } 1990 1991 if (!segzio_fromheap) { 1992 size_t size; 1993 size_t physmem_b = mmu_ptob(physmem); 1994 1995 /* size is in bytes, segziosize is in pages */ 1996 if (segziosize == 0) { 1997 size = physmem_b; 1998 } else { 1999 size = mmu_ptob(segziosize); 2000 } 2001 2002 if (size < SEGZIOMINSIZE) { 2003 size = SEGZIOMINSIZE; 2004 } else if (size > SEGZIOMAXSIZE) { 2005 size = SEGZIOMAXSIZE; 2006 /* 2007 * On 64-bit x86, we only have 2TB of KVA. This exists 2008 * for parity with x86. 2009 * 2010 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2011 * doesn't consume all of KVA. However, if we have a 2012 * system that has more thant 512gb of physical memory, 2013 * we can actually consume about half of the difference 2014 * between 512gb and the rest of the available physical 2015 * memory. 2016 */ 2017 if (physmem_b > SEGZIOMAXSIZE) { 2018 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2019 } 2020 } 2021 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2022 /* put the base of the ZIO segment after the kpm segment */ 2023 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2024 PRM_DEBUG(segziosize); 2025 PRM_DEBUG(segzio_base); 2026 2027 /* 2028 * On some platforms, kvm_init is called after the kpm 2029 * sizes have been determined. On SPARC, kvm_init is called 2030 * before, so we have to attach the kzioseg after kvm is 2031 * initialized, otherwise we'll try to allocate from the boot 2032 * area since the kernel heap hasn't yet been configured. 2033 */ 2034 rw_enter(&kas.a_lock, RW_WRITER); 2035 2036 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2037 &kzioseg); 2038 (void) segkmem_zio_create(&kzioseg); 2039 2040 /* create zio area covering new segment */ 2041 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2042 2043 rw_exit(&kas.a_lock); 2044 } 2045 2046 2047 /* 2048 * Now create generic mapping segment. This mapping 2049 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2050 * virtual address is greater than the amount of free 2051 * memory that is available, then we trim back the 2052 * segment size to that amount 2053 */ 2054 va = (caddr_t)SEGMAPBASE; 2055 2056 /* 2057 * 1201049: segkmap base address must be MAXBSIZE aligned 2058 */ 2059 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2060 2061 /* 2062 * Set size of segmap to percentage of freemem at boot, 2063 * but stay within the allowable range 2064 * Note we take percentage before converting from pages 2065 * to bytes to avoid an overflow on 32-bit kernels. 2066 */ 2067 i = mmu_ptob((freemem * segmap_percent) / 100); 2068 2069 if (i < MINMAPSIZE) 2070 i = MINMAPSIZE; 2071 2072 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2073 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2074 2075 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2076 2077 rw_enter(&kas.a_lock, RW_WRITER); 2078 if (seg_attach(&kas, va, i, segkmap) < 0) 2079 cmn_err(CE_PANIC, "cannot attach segkmap"); 2080 2081 a.prot = PROT_READ | PROT_WRITE; 2082 a.shmsize = shm_alignment; 2083 a.nfreelist = 0; /* use segmap driver defaults */ 2084 2085 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2086 panic("segmap_create segkmap"); 2087 rw_exit(&kas.a_lock); 2088 2089 segdev_init(); 2090 } 2091 2092 static void 2093 startup_end(void) 2094 { 2095 if ((caddr_t)memlist > (caddr_t)memlist_end) 2096 panic("memlist overflow 2"); 2097 memlist_free_block((caddr_t)memlist, 2098 ((caddr_t)memlist_end - (caddr_t)memlist)); 2099 memlist = NULL; 2100 2101 /* enable page_relocation since OBP is now done */ 2102 page_relocate_ready = 1; 2103 2104 /* 2105 * Perform tasks that get done after most of the VM 2106 * initialization has been done but before the clock 2107 * and other devices get started. 2108 */ 2109 kern_setup1(); 2110 2111 /* 2112 * Intialize the VM arenas for allocating physically 2113 * contiguus memory chunk for interrupt queues snd 2114 * allocate/register boot cpu's queues, if any and 2115 * allocate dump buffer for sun4v systems to store 2116 * extra crash information during crash dump 2117 */ 2118 contig_mem_init(); 2119 mach_descrip_init(); 2120 2121 if (cpu_intrq_setup(CPU)) { 2122 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2123 } 2124 cpu_intrq_register(CPU); 2125 mach_htraptrace_setup(CPU->cpu_id); 2126 mach_htraptrace_configure(CPU->cpu_id); 2127 mach_dump_buffer_init(); 2128 2129 /* 2130 * Initialize interrupt related stuff 2131 */ 2132 cpu_intr_alloc(CPU, NINTR_THREADS); 2133 2134 (void) splzs(); /* allow hi clock ints but not zs */ 2135 2136 /* 2137 * Initialize errors. 2138 */ 2139 error_init(); 2140 2141 /* 2142 * Note that we may have already used kernel bcopy before this 2143 * point - but if you really care about this, adb the use_hw_* 2144 * variables to 0 before rebooting. 2145 */ 2146 mach_hw_copy_limit(); 2147 2148 /* 2149 * Install the "real" preemption guards before DDI services 2150 * are available. 2151 */ 2152 (void) prom_set_preprom(kern_preprom); 2153 (void) prom_set_postprom(kern_postprom); 2154 CPU->cpu_m.mutex_ready = 1; 2155 2156 /* 2157 * Initialize segnf (kernel support for non-faulting loads). 2158 */ 2159 segnf_init(); 2160 2161 /* 2162 * Configure the root devinfo node. 2163 */ 2164 configure(); /* set up devices */ 2165 mach_cpu_halt_idle(); 2166 } 2167 2168 2169 void 2170 post_startup(void) 2171 { 2172 #ifdef PTL1_PANIC_DEBUG 2173 extern void init_ptl1_thread(void); 2174 #endif /* PTL1_PANIC_DEBUG */ 2175 extern void abort_sequence_init(void); 2176 2177 /* 2178 * Set the system wide, processor-specific flags to be passed 2179 * to userland via the aux vector for performance hints and 2180 * instruction set extensions. 2181 */ 2182 bind_hwcap(); 2183 2184 /* 2185 * Startup memory scrubber (if any) 2186 */ 2187 mach_memscrub(); 2188 2189 /* 2190 * Allocate soft interrupt to handle abort sequence. 2191 */ 2192 abort_sequence_init(); 2193 2194 /* 2195 * Configure the rest of the system. 2196 * Perform forceloading tasks for /etc/system. 2197 */ 2198 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2199 /* 2200 * ON4.0: Force /proc module in until clock interrupt handle fixed 2201 * ON4.0: This must be fixed or restated in /etc/systems. 2202 */ 2203 (void) modload("fs", "procfs"); 2204 2205 /* load machine class specific drivers */ 2206 load_mach_drivers(); 2207 2208 /* load platform specific drivers */ 2209 if (&load_platform_drivers) 2210 load_platform_drivers(); 2211 2212 /* load vis simulation module, if we are running w/fpu off */ 2213 if (!fpu_exists) { 2214 if (modload("misc", "vis") == -1) 2215 halt("Can't load vis"); 2216 } 2217 2218 mach_fpras(); 2219 2220 maxmem = freemem; 2221 2222 #ifdef PTL1_PANIC_DEBUG 2223 init_ptl1_thread(); 2224 #endif /* PTL1_PANIC_DEBUG */ 2225 } 2226 2227 #ifdef PTL1_PANIC_DEBUG 2228 int ptl1_panic_test = 0; 2229 int ptl1_panic_xc_one_test = 0; 2230 int ptl1_panic_xc_all_test = 0; 2231 int ptl1_panic_xt_one_test = 0; 2232 int ptl1_panic_xt_all_test = 0; 2233 kthread_id_t ptl1_thread_p = NULL; 2234 kcondvar_t ptl1_cv; 2235 kmutex_t ptl1_mutex; 2236 int ptl1_recurse_count_threshold = 0x40; 2237 int ptl1_recurse_trap_threshold = 0x3d; 2238 extern void ptl1_recurse(int, int); 2239 extern void ptl1_panic_xt(int, int); 2240 2241 /* 2242 * Called once per second by timeout() to wake up 2243 * the ptl1_panic thread to see if it should cause 2244 * a trap to the ptl1_panic() code. 2245 */ 2246 /* ARGSUSED */ 2247 static void 2248 ptl1_wakeup(void *arg) 2249 { 2250 mutex_enter(&ptl1_mutex); 2251 cv_signal(&ptl1_cv); 2252 mutex_exit(&ptl1_mutex); 2253 } 2254 2255 /* 2256 * ptl1_panic cross call function: 2257 * Needed because xc_one() and xc_some() can pass 2258 * 64 bit args but ptl1_recurse() expects ints. 2259 */ 2260 static void 2261 ptl1_panic_xc(void) 2262 { 2263 ptl1_recurse(ptl1_recurse_count_threshold, 2264 ptl1_recurse_trap_threshold); 2265 } 2266 2267 /* 2268 * The ptl1 thread waits for a global flag to be set 2269 * and uses the recurse thresholds to set the stack depth 2270 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2271 * or indirectly via the cross call and cross trap functions. 2272 * 2273 * This is useful testing stack overflows and normal 2274 * ptl1_panic() states with a know stack frame. 2275 * 2276 * ptl1_recurse() is an asm function in ptl1_panic.s that 2277 * sets the {In, Local, Out, and Global} registers to a 2278 * know state on the stack and just prior to causing a 2279 * test ptl1_panic trap. 2280 */ 2281 static void 2282 ptl1_thread(void) 2283 { 2284 mutex_enter(&ptl1_mutex); 2285 while (ptl1_thread_p) { 2286 cpuset_t other_cpus; 2287 int cpu_id; 2288 int my_cpu_id; 2289 int target_cpu_id; 2290 int target_found; 2291 2292 if (ptl1_panic_test) { 2293 ptl1_recurse(ptl1_recurse_count_threshold, 2294 ptl1_recurse_trap_threshold); 2295 } 2296 2297 /* 2298 * Find potential targets for x-call and x-trap, 2299 * if any exist while preempt is disabled we 2300 * start a ptl1_panic if requested via a 2301 * globals. 2302 */ 2303 kpreempt_disable(); 2304 my_cpu_id = CPU->cpu_id; 2305 other_cpus = cpu_ready_set; 2306 CPUSET_DEL(other_cpus, CPU->cpu_id); 2307 target_found = 0; 2308 if (!CPUSET_ISNULL(other_cpus)) { 2309 /* 2310 * Pick the first one 2311 */ 2312 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2313 if (cpu_id == my_cpu_id) 2314 continue; 2315 2316 if (CPU_XCALL_READY(cpu_id)) { 2317 target_cpu_id = cpu_id; 2318 target_found = 1; 2319 break; 2320 } 2321 } 2322 ASSERT(target_found); 2323 2324 if (ptl1_panic_xc_one_test) { 2325 xc_one(target_cpu_id, 2326 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2327 } 2328 if (ptl1_panic_xc_all_test) { 2329 xc_some(other_cpus, 2330 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2331 } 2332 if (ptl1_panic_xt_one_test) { 2333 xt_one(target_cpu_id, 2334 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2335 } 2336 if (ptl1_panic_xt_all_test) { 2337 xt_some(other_cpus, 2338 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2339 } 2340 } 2341 kpreempt_enable(); 2342 (void) timeout(ptl1_wakeup, NULL, hz); 2343 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2344 } 2345 mutex_exit(&ptl1_mutex); 2346 } 2347 2348 /* 2349 * Called during early startup to create the ptl1_thread 2350 */ 2351 void 2352 init_ptl1_thread(void) 2353 { 2354 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2355 &p0, TS_RUN, 0); 2356 } 2357 #endif /* PTL1_PANIC_DEBUG */ 2358 2359 2360 static void 2361 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2362 { 2363 struct memlist *new; 2364 2365 new = *memlistp; 2366 new->address = start; 2367 new->size = len; 2368 *memlistp = new + 1; 2369 } 2370 2371 /* 2372 * Add to a memory list. 2373 * start = start of new memory segment 2374 * len = length of new memory segment in bytes 2375 * memlistp = pointer to array of available memory segment structures 2376 * curmemlistp = memory list to which to add segment. 2377 */ 2378 static void 2379 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2380 struct memlist **curmemlistp) 2381 { 2382 struct memlist *new = *memlistp; 2383 2384 memlist_new(start, len, memlistp); 2385 memlist_insert(new, curmemlistp); 2386 } 2387 2388 static int 2389 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2390 { 2391 int nseg; 2392 size_t memseg_sz; 2393 struct memseg *msp; 2394 2395 /* 2396 * The memseg list is for the chunks of physical memory that 2397 * will be managed by the vm system. The number calculated is 2398 * a guess as boot may fragment it more when memory allocations 2399 * are made before kphysm_init(). 2400 */ 2401 memseg_sz = (avail + 10) * sizeof (struct memseg); 2402 memseg_sz = roundup(memseg_sz, PAGESIZE); 2403 nseg = memseg_sz / sizeof (struct memseg); 2404 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2405 if (msp == NULL) 2406 return (1); 2407 PRM_DEBUG(memseg_free); 2408 2409 while (nseg--) { 2410 msp->next = memseg_free; 2411 memseg_free = msp; 2412 msp++; 2413 } 2414 return (0); 2415 } 2416 2417 /* 2418 * In the case of architectures that support dynamic addition of 2419 * memory at run-time there are two cases where memsegs need to 2420 * be initialized and added to the memseg list. 2421 * 1) memsegs that are constructed at startup. 2422 * 2) memsegs that are constructed at run-time on 2423 * hot-plug capable architectures. 2424 * This code was originally part of the function kphysm_init(). 2425 */ 2426 2427 static void 2428 memseg_list_add(struct memseg *memsegp) 2429 { 2430 struct memseg **prev_memsegp; 2431 pgcnt_t num; 2432 2433 /* insert in memseg list, decreasing number of pages order */ 2434 2435 num = MSEG_NPAGES(memsegp); 2436 2437 for (prev_memsegp = &memsegs; *prev_memsegp; 2438 prev_memsegp = &((*prev_memsegp)->next)) { 2439 if (num > MSEG_NPAGES(*prev_memsegp)) 2440 break; 2441 } 2442 2443 memsegp->next = *prev_memsegp; 2444 *prev_memsegp = memsegp; 2445 2446 if (kpm_enable) { 2447 memsegp->nextpa = (memsegp->next) ? 2448 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2449 2450 if (prev_memsegp != &memsegs) { 2451 struct memseg *msp; 2452 msp = (struct memseg *)((caddr_t)prev_memsegp - 2453 offsetof(struct memseg, next)); 2454 msp->nextpa = va_to_pa(memsegp); 2455 } else { 2456 memsegspa = va_to_pa(memsegs); 2457 } 2458 } 2459 } 2460 2461 /* 2462 * PSM add_physmem_cb(). US-II and newer processors have some 2463 * flavor of the prefetch capability implemented. We exploit 2464 * this capability for optimum performance. 2465 */ 2466 #define PREFETCH_BYTES 64 2467 2468 void 2469 add_physmem_cb(page_t *pp, pfn_t pnum) 2470 { 2471 extern void prefetch_page_w(void *); 2472 2473 pp->p_pagenum = pnum; 2474 2475 /* 2476 * Prefetch one more page_t into E$. To prevent future 2477 * mishaps with the sizeof(page_t) changing on us, we 2478 * catch this on debug kernels if we can't bring in the 2479 * entire hpage with 2 PREFETCH_BYTES reads. See 2480 * also, sun4u/cpu/cpu_module.c 2481 */ 2482 /*LINTED*/ 2483 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2484 prefetch_page_w((char *)pp); 2485 } 2486 2487 /* 2488 * Find memseg with given pfn 2489 */ 2490 static struct memseg * 2491 memseg_find(pfn_t base, pfn_t *next) 2492 { 2493 struct memseg *seg; 2494 2495 if (next != NULL) 2496 *next = LONG_MAX; 2497 for (seg = memsegs; seg != NULL; seg = seg->next) { 2498 if (base >= seg->pages_base && base < seg->pages_end) 2499 return (seg); 2500 if (next != NULL && seg->pages_base > base && 2501 seg->pages_base < *next) 2502 *next = seg->pages_base; 2503 } 2504 return (NULL); 2505 } 2506 2507 extern struct vnode prom_ppages; 2508 2509 /* 2510 * Put page allocated by OBP on prom_ppages 2511 */ 2512 static void 2513 kphysm_erase(uint64_t addr, uint64_t len) 2514 { 2515 struct page *pp; 2516 struct memseg *seg; 2517 pfn_t base = btop(addr), next; 2518 pgcnt_t num = btop(len); 2519 2520 while (num != 0) { 2521 pgcnt_t off, left; 2522 2523 seg = memseg_find(base, &next); 2524 if (seg == NULL) { 2525 if (next == LONG_MAX) 2526 break; 2527 left = MIN(next - base, num); 2528 base += left, num -= left; 2529 continue; 2530 } 2531 off = base - seg->pages_base; 2532 pp = seg->pages + off; 2533 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2534 while (num != left) { 2535 /* 2536 * init it, lock it, and hashin on prom_pages vp. 2537 * 2538 * XXX vnode offsets on the prom_ppages vnode 2539 * are page numbers (gack) for >32 bit 2540 * physical memory machines. 2541 */ 2542 add_physmem_cb(pp, base); 2543 if (page_trylock(pp, SE_EXCL) == 0) 2544 cmn_err(CE_PANIC, "prom page locked"); 2545 (void) page_hashin(pp, &prom_ppages, 2546 (offset_t)base, NULL); 2547 (void) page_pp_lock(pp, 0, 1); 2548 pp++, base++, num--; 2549 } 2550 } 2551 } 2552 2553 static page_t *ppnext; 2554 static pgcnt_t ppleft; 2555 2556 static void *kpm_ppnext; 2557 static pgcnt_t kpm_ppleft; 2558 2559 /* 2560 * Create a memseg 2561 */ 2562 static void 2563 kphysm_memseg(uint64_t addr, uint64_t len) 2564 { 2565 pfn_t base = btop(addr); 2566 pgcnt_t num = btop(len); 2567 struct memseg *seg; 2568 2569 seg = memseg_free; 2570 memseg_free = seg->next; 2571 ASSERT(seg != NULL); 2572 2573 seg->pages = ppnext; 2574 seg->epages = ppnext + num; 2575 seg->pages_base = base; 2576 seg->pages_end = base + num; 2577 ppnext += num; 2578 ppleft -= num; 2579 2580 if (kpm_enable) { 2581 pgcnt_t kpnum = ptokpmpr(num); 2582 2583 if (kpnum > kpm_ppleft) 2584 panic("kphysm_memseg: kpm_pp overflow"); 2585 seg->pagespa = va_to_pa(seg->pages); 2586 seg->epagespa = va_to_pa(seg->epages); 2587 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2588 seg->kpm_nkpmpgs = kpnum; 2589 /* 2590 * In the kpm_smallpage case, the kpm array 2591 * is 1-1 wrt the page array 2592 */ 2593 if (kpm_smallpages) { 2594 kpm_spage_t *kpm_pp = kpm_ppnext; 2595 2596 kpm_ppnext = kpm_pp + kpnum; 2597 seg->kpm_spages = kpm_pp; 2598 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2599 } else { 2600 kpm_page_t *kpm_pp = kpm_ppnext; 2601 2602 kpm_ppnext = kpm_pp + kpnum; 2603 seg->kpm_pages = kpm_pp; 2604 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2605 /* ASSERT no kpm overlaps */ 2606 ASSERT( 2607 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2608 ASSERT(memseg_find( 2609 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2610 } 2611 kpm_ppleft -= num; 2612 } 2613 2614 memseg_list_add(seg); 2615 } 2616 2617 /* 2618 * Add range to free list 2619 */ 2620 void 2621 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2622 { 2623 struct page *pp; 2624 struct memseg *seg; 2625 pfn_t base = btop(addr); 2626 pgcnt_t num = btop(len); 2627 2628 seg = memseg_find(base, NULL); 2629 ASSERT(seg != NULL); 2630 pp = seg->pages + (base - seg->pages_base); 2631 2632 if (reclaim) { 2633 struct page *rpp = pp; 2634 struct page *lpp = pp + num; 2635 2636 /* 2637 * page should be locked on prom_ppages 2638 * unhash and unlock it 2639 */ 2640 while (rpp < lpp) { 2641 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &prom_ppages); 2642 page_pp_unlock(rpp, 0, 1); 2643 page_hashout(rpp, NULL); 2644 page_unlock(rpp); 2645 rpp++; 2646 } 2647 } 2648 2649 /* 2650 * add_physmem() initializes the PSM part of the page 2651 * struct by calling the PSM back with add_physmem_cb(). 2652 * In addition it coalesces pages into larger pages as 2653 * it initializes them. 2654 */ 2655 add_physmem(pp, num, base); 2656 } 2657 2658 /* 2659 * kphysm_init() tackles the problem of initializing physical memory. 2660 */ 2661 static void 2662 kphysm_init(void) 2663 { 2664 struct memlist *pmem; 2665 2666 ASSERT(page_hash != NULL && page_hashsz != 0); 2667 2668 ppnext = pp_base; 2669 ppleft = npages; 2670 kpm_ppnext = kpm_pp_base; 2671 kpm_ppleft = kpm_npages; 2672 2673 /* 2674 * installed pages not on nopp_memlist go in memseg list 2675 */ 2676 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2677 2678 /* 2679 * Free the avail list 2680 */ 2681 for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 2682 kphysm_add(pmem->address, pmem->size, 0); 2683 2684 /* 2685 * Erase pages that aren't available 2686 */ 2687 diff_memlists(phys_install, phys_avail, kphysm_erase); 2688 2689 build_pfn_hash(); 2690 } 2691 2692 /* 2693 * Kernel VM initialization. 2694 * Assumptions about kernel address space ordering: 2695 * (1) gap (user space) 2696 * (2) kernel text 2697 * (3) kernel data/bss 2698 * (4) gap 2699 * (5) kernel data structures 2700 * (6) gap 2701 * (7) debugger (optional) 2702 * (8) monitor 2703 * (9) gap (possibly null) 2704 * (10) dvma 2705 * (11) devices 2706 */ 2707 static void 2708 kvm_init(void) 2709 { 2710 /* 2711 * Put the kernel segments in kernel address space. 2712 */ 2713 rw_enter(&kas.a_lock, RW_WRITER); 2714 as_avlinit(&kas); 2715 2716 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2717 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2718 (void) segkmem_create(&ktextseg); 2719 2720 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2721 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2722 (void) segkmem_create(&ktexthole); 2723 2724 (void) seg_attach(&kas, (caddr_t)valloc_base, 2725 (size_t)(econtig32 - valloc_base), &kvalloc); 2726 (void) segkmem_create(&kvalloc); 2727 2728 if (kmem64_base) { 2729 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2730 (size_t)(kmem64_end - kmem64_base), &kmem64); 2731 (void) segkmem_create(&kmem64); 2732 } 2733 2734 /* 2735 * We're about to map out /boot. This is the beginning of the 2736 * system resource management transition. We can no longer 2737 * call into /boot for I/O or memory allocations. 2738 */ 2739 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2740 (void) segkmem_create(&kvseg); 2741 hblk_alloc_dynamic = 1; 2742 2743 /* 2744 * we need to preallocate pages for DR operations before enabling large 2745 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2746 */ 2747 memseg_remap_init(); 2748 2749 /* at this point we are ready to use large page heap */ 2750 segkmem_heap_lp_init(); 2751 2752 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2753 &kvseg32); 2754 (void) segkmem_create(&kvseg32); 2755 2756 /* 2757 * Create a segment for the debugger. 2758 */ 2759 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2760 (void) segkmem_create(&kdebugseg); 2761 2762 rw_exit(&kas.a_lock); 2763 } 2764 2765 char obp_tte_str[] = 2766 "h# %x constant MMU_PAGESHIFT " 2767 "h# %x constant TTE8K " 2768 "h# %x constant SFHME_SIZE " 2769 "h# %x constant SFHME_TTE " 2770 "h# %x constant HMEBLK_TAG " 2771 "h# %x constant HMEBLK_NEXT " 2772 "h# %x constant HMEBLK_MISC " 2773 "h# %x constant HMEBLK_HME1 " 2774 "h# %x constant NHMENTS " 2775 "h# %x constant HBLK_SZMASK " 2776 "h# %x constant HBLK_RANGE_SHIFT " 2777 "h# %x constant HMEBP_HBLK " 2778 "h# %x constant HMEBUCKET_SIZE " 2779 "h# %x constant HTAG_SFMMUPSZ " 2780 "h# %x constant HTAG_BSPAGE_SHIFT " 2781 "h# %x constant HTAG_REHASH_SHIFT " 2782 "h# %x constant SFMMU_INVALID_SHMERID " 2783 "h# %x constant mmu_hashcnt " 2784 "h# %p constant uhme_hash " 2785 "h# %p constant khme_hash " 2786 "h# %x constant UHMEHASH_SZ " 2787 "h# %x constant KHMEHASH_SZ " 2788 "h# %p constant KCONTEXT " 2789 "h# %p constant KHATID " 2790 "h# %x constant ASI_MEM " 2791 2792 ": PHYS-X@ ( phys -- data ) " 2793 " ASI_MEM spacex@ " 2794 "; " 2795 2796 ": PHYS-W@ ( phys -- data ) " 2797 " ASI_MEM spacew@ " 2798 "; " 2799 2800 ": PHYS-L@ ( phys -- data ) " 2801 " ASI_MEM spaceL@ " 2802 "; " 2803 2804 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2805 " 3 * MMU_PAGESHIFT + " 2806 "; " 2807 2808 ": TTE_IS_VALID ( ttep -- flag ) " 2809 " PHYS-X@ 0< " 2810 "; " 2811 2812 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2813 " dup TTE8K = if " 2814 " drop HBLK_RANGE_SHIFT " 2815 " else " 2816 " TTE_PAGE_SHIFT " 2817 " then " 2818 "; " 2819 2820 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2821 " tuck >> swap MMU_PAGESHIFT - << " 2822 "; " 2823 2824 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2825 " >> over xor swap ( hash sfmmup ) " 2826 " KHATID <> if ( hash ) " 2827 " UHMEHASH_SZ and ( bucket ) " 2828 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2829 " else ( hash ) " 2830 " KHMEHASH_SZ and ( bucket ) " 2831 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2832 " then ( hmebp ) " 2833 "; " 2834 2835 ": HME_HASH_TABLE_SEARCH " 2836 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2837 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2838 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2839 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2840 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2841 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2842 " else " 2843 " hmeblk_next + phys-x@ false " 2844 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2845 " then " 2846 " else " 2847 " hmeblk_next + phys-x@ false " 2848 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2849 " then " 2850 " else " 2851 " true " 2852 " then " 2853 " until r> drop " 2854 "; " 2855 2856 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2857 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2858 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2859 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2860 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2861 "; " 2862 2863 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2864 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2865 " TTE8K = if ( hmeblkp addr ) " 2866 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2867 " else ( hmeblkp addr ) " 2868 " drop 0 ( hmeblkp 0 ) " 2869 " then ( hmeblkp hme-index ) " 2870 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2871 " SFHME_TTE + ( ttep ) " 2872 "; " 2873 2874 ": unix-tte ( addr cnum -- false | tte-data true ) " 2875 " KCONTEXT = if ( addr ) " 2876 " KHATID ( addr khatid ) " 2877 " else ( addr ) " 2878 " drop false exit ( false ) " 2879 " then " 2880 " ( addr khatid ) " 2881 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2882 " 2dup swap i HME_HASH_SHIFT " 2883 "( addr sfmmup sfmmup addr hmeshift ) " 2884 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2885 " over i 4 pick " 2886 "( addr sfmmup hmebp sfmmup rehash addr ) " 2887 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2888 " HME_HASH_TABLE_SEARCH " 2889 "( addr sfmmup { null | hmeblkp } ) " 2890 " ?dup if ( addr sfmmup hmeblkp ) " 2891 " nip swap HBLK_TO_TTEP ( ttep ) " 2892 " dup TTE_IS_VALID if ( valid-ttep ) " 2893 " PHYS-X@ true ( tte-data true ) " 2894 " else ( invalid-tte ) " 2895 " drop false ( false ) " 2896 " then ( false | tte-data true ) " 2897 " unloop exit ( false | tte-data true ) " 2898 " then ( addr sfmmup ) " 2899 " loop ( addr sfmmup ) " 2900 " 2drop false ( false ) " 2901 "; " 2902 ; 2903 2904 void 2905 create_va_to_tte(void) 2906 { 2907 char *bp; 2908 extern int khmehash_num, uhmehash_num; 2909 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2910 2911 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2912 2913 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2914 2915 /* 2916 * Teach obp how to parse our sw ttes. 2917 */ 2918 (void) sprintf(bp, obp_tte_str, 2919 MMU_PAGESHIFT, 2920 TTE8K, 2921 sizeof (struct sf_hment), 2922 OFFSET(struct sf_hment, hme_tte), 2923 OFFSET(struct hme_blk, hblk_tag), 2924 OFFSET(struct hme_blk, hblk_nextpa), 2925 OFFSET(struct hme_blk, hblk_misc), 2926 OFFSET(struct hme_blk, hblk_hme), 2927 NHMENTS, 2928 HBLK_SZMASK, 2929 HBLK_RANGE_SHIFT, 2930 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2931 sizeof (struct hmehash_bucket), 2932 HTAG_SFMMUPSZ, 2933 HTAG_BSPAGE_SHIFT, 2934 HTAG_REHASH_SHIFT, 2935 SFMMU_INVALID_SHMERID, 2936 mmu_hashcnt, 2937 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2938 (caddr_t)va_to_pa((caddr_t)khme_hash), 2939 UHMEHASH_SZ, 2940 KHMEHASH_SZ, 2941 KCONTEXT, 2942 KHATID, 2943 ASI_MEM); 2944 prom_interpret(bp, 0, 0, 0, 0, 0); 2945 2946 kobj_free(bp, MMU_PAGESIZE); 2947 } 2948 2949 void 2950 install_va_to_tte(void) 2951 { 2952 /* 2953 * advise prom that he can use unix-tte 2954 */ 2955 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2956 } 2957 2958 /* 2959 * Here we add "device-type=console" for /os-io node, for currently 2960 * our kernel console output only supports displaying text and 2961 * performing cursor-positioning operations (through kernel framebuffer 2962 * driver) and it doesn't support other functionalities required for a 2963 * standard "display" device as specified in 1275 spec. The main missing 2964 * interface defined by the 1275 spec is "draw-logo". 2965 * also see the comments above prom_stdout_is_framebuffer(). 2966 */ 2967 static char *create_node = 2968 "\" /\" find-device " 2969 "new-device " 2970 "\" os-io\" device-name " 2971 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 2972 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2973 " 2>r swap 2 2r> ['] $callback catch if " 2974 " 2drop 3drop 0 " 2975 " then " 2976 "; " 2977 ": read ( adr,len -- #read ) " 2978 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2979 " ( retN ... ret1 N ) " 2980 " ?dup if " 2981 " swap >r 1- 0 ?do drop loop r> " 2982 " else " 2983 " -2 " 2984 " then " 2985 "; " 2986 ": write ( adr,len -- #written ) " 2987 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2988 " ( retN ... ret1 N ) " 2989 " ?dup if " 2990 " swap >r 1- 0 ?do drop loop r> " 2991 " else " 2992 " 0 " 2993 " then " 2994 "; " 2995 ": poll-tty ( -- ) ; " 2996 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2997 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2998 ": cb-give/take ( $method -- ) " 2999 " 0 -rot ['] $callback catch ?dup if " 3000 " >r 2drop 2drop r> throw " 3001 " else " 3002 " 0 ?do drop loop " 3003 " then " 3004 "; " 3005 ": give ( -- ) \" exit-input\" cb-give/take ; " 3006 ": take ( -- ) \" enter-input\" cb-give/take ; " 3007 ": open ( -- ok? ) true ; " 3008 ": close ( -- ) ; " 3009 "finish-device " 3010 "device-end "; 3011 3012 /* 3013 * Create the OBP input/output node (FCode serial driver). 3014 * It is needed for both USB console keyboard and for 3015 * the kernel terminal emulator. It is too early to check for a 3016 * kernel console compatible framebuffer now, so we create this 3017 * so that we're ready if we need to enable kernel terminal emulation. 3018 * 3019 * When the USB software takes over the input device at the time 3020 * consconfig runs, OBP's stdin is redirected to this node. 3021 * Whenever the FORTH user interface is used after this switch, 3022 * the node will call back into the kernel for console input. 3023 * If a serial device such as ttya or a UART with a Type 5 keyboard 3024 * attached is used, OBP takes over the serial device when the system 3025 * goes to the debugger after the system is booted. This sharing 3026 * of the relatively simple serial device is difficult but possible. 3027 * Sharing the USB host controller is impossible due its complexity. 3028 * 3029 * Similarly to USB keyboard input redirection, after consconfig_dacf 3030 * configures a kernel console framebuffer as the standard output 3031 * device, OBP's stdout is switched to to vector through the 3032 * /os-io node into the kernel terminal emulator. 3033 */ 3034 static void 3035 startup_create_io_node(void) 3036 { 3037 prom_interpret(create_node, 0, 0, 0, 0, 0); 3038 } 3039 3040 3041 static void 3042 do_prom_version_check(void) 3043 { 3044 int i; 3045 pnode_t node; 3046 char buf[64]; 3047 static char drev[] = "Down-rev firmware detected%s\n" 3048 "\tPlease upgrade to the following minimum version:\n" 3049 "\t\t%s\n"; 3050 3051 i = prom_version_check(buf, sizeof (buf), &node); 3052 3053 if (i == PROM_VER64_OK) 3054 return; 3055 3056 if (i == PROM_VER64_UPGRADE) { 3057 cmn_err(CE_WARN, drev, "", buf); 3058 3059 #ifdef DEBUG 3060 prom_enter_mon(); /* Type 'go' to continue */ 3061 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3062 return; 3063 #else 3064 halt(0); 3065 #endif 3066 } 3067 3068 /* 3069 * The other possibility is that this is a server running 3070 * good firmware, but down-rev firmware was detected on at 3071 * least one other cpu board. We just complain if we see 3072 * that. 3073 */ 3074 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3075 } 3076 3077 3078 /* 3079 * Must be defined in platform dependent code. 3080 */ 3081 extern caddr_t modtext; 3082 extern size_t modtext_sz; 3083 extern caddr_t moddata; 3084 3085 #define HEAPTEXT_ARENA(addr) \ 3086 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3087 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3088 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3089 3090 #define HEAPTEXT_OVERSIZED(addr) \ 3091 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3092 3093 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3094 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3095 kmutex_t texthole_lock; 3096 3097 char kern_bootargs[OBP_MAXPATHLEN]; 3098 3099 void 3100 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3101 { 3102 uintptr_t addr, limit; 3103 3104 addr = HEAPTEXT_BASE; 3105 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3106 3107 /* 3108 * Before we initialize the text_arena, we want to punch holes in the 3109 * underlying heaptext_arena. This guarantees that for any text 3110 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3111 */ 3112 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3113 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3114 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3115 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3116 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3117 } 3118 3119 /* 3120 * Allocate one page at the oversize to break up the text region 3121 * from the oversized region. 3122 */ 3123 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3124 (void *)limit, (void *)(limit + PAGESIZE), 3125 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3126 3127 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3128 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3129 heaptext_arena, 0, VM_SLEEP); 3130 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3131 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3132 } 3133 3134 caddr_t 3135 kobj_text_alloc(vmem_t *arena, size_t size) 3136 { 3137 caddr_t rval, better; 3138 3139 /* 3140 * First, try a sleeping allocation. 3141 */ 3142 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3143 3144 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3145 return (rval); 3146 3147 /* 3148 * We didn't get the area that we wanted. We're going to try to do an 3149 * allocation with explicit constraints. 3150 */ 3151 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3152 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3153 VM_NOSLEEP | VM_BESTFIT); 3154 3155 if (better != NULL) { 3156 /* 3157 * That worked. Free our first attempt and return. 3158 */ 3159 vmem_free(arena, rval, size); 3160 return (better); 3161 } 3162 3163 /* 3164 * That didn't work; we'll have to return our first attempt. 3165 */ 3166 return (rval); 3167 } 3168 3169 caddr_t 3170 kobj_texthole_alloc(caddr_t addr, size_t size) 3171 { 3172 int arena = HEAPTEXT_ARENA(addr); 3173 char c[30]; 3174 uintptr_t base; 3175 3176 if (HEAPTEXT_OVERSIZED(addr)) { 3177 /* 3178 * If this is an oversized allocation, there is no text hole 3179 * available for it; return NULL. 3180 */ 3181 return (NULL); 3182 } 3183 3184 mutex_enter(&texthole_lock); 3185 3186 if (texthole_arena[arena] == NULL) { 3187 ASSERT(texthole_source[arena] == NULL); 3188 3189 if (arena == 0) { 3190 texthole_source[0] = vmem_create("module_text_holesrc", 3191 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3192 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3193 0, VM_SLEEP); 3194 } else { 3195 base = HEAPTEXT_BASE + 3196 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3197 3198 (void) snprintf(c, sizeof (c), 3199 "heaptext_holesrc_%d", arena); 3200 3201 texthole_source[arena] = vmem_create(c, (void *)base, 3202 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3203 0, VM_SLEEP); 3204 } 3205 3206 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3207 3208 texthole_arena[arena] = vmem_create(c, NULL, 0, 3209 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3210 texthole_source[arena], 0, VM_SLEEP); 3211 } 3212 3213 mutex_exit(&texthole_lock); 3214 3215 ASSERT(texthole_arena[arena] != NULL); 3216 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3217 return (vmem_alloc(texthole_arena[arena], size, 3218 VM_BESTFIT | VM_NOSLEEP)); 3219 } 3220 3221 void 3222 kobj_texthole_free(caddr_t addr, size_t size) 3223 { 3224 int arena = HEAPTEXT_ARENA(addr); 3225 3226 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3227 ASSERT(texthole_arena[arena] != NULL); 3228 vmem_free(texthole_arena[arena], addr, size); 3229 } 3230 3231 void 3232 release_bootstrap(void) 3233 { 3234 if (&cif_init) 3235 cif_init(); 3236 } 3237