1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/intreg.h> 43 #include <sys/ivintr.h> 44 #include <sys/fpu/fpusystm.h> 45 #include <sys/iommutsb.h> 46 #include <vm/vm_dep.h> 47 #include <vm/seg_dev.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_kpm.h> 50 #include <vm/seg_map.h> 51 #include <vm/seg_kp.h> 52 #include <sys/sysconf.h> 53 #include <vm/hat_sfmmu.h> 54 #include <sys/kobj.h> 55 #include <sys/sun4asi.h> 56 #include <sys/clconf.h> 57 #include <sys/platform_module.h> 58 #include <sys/panic.h> 59 #include <sys/cpu_sgnblk_defs.h> 60 #include <sys/clock.h> 61 #include <sys/cmn_err.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 69 extern void setup_trap_table(void); 70 extern void cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern void mach_dump_buffer_init(void); 74 extern void mach_descrip_init(void); 75 extern void mach_descrip_startup_fini(void); 76 extern void mach_memscrub(void); 77 extern void mach_fpras(void); 78 extern void mach_cpu_halt_idle(void); 79 extern void mach_hw_copy_limit(void); 80 extern void load_mach_drivers(void); 81 extern void load_tod_module(void); 82 #pragma weak load_tod_module 83 84 extern int ndata_alloc_mmfsa(struct memlist *ndata); 85 #pragma weak ndata_alloc_mmfsa 86 87 extern void cif_init(void); 88 #pragma weak cif_init 89 90 extern void parse_idprom(void); 91 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 92 extern void mem_config_init(void); 93 extern void memseg_remap_init(void); 94 95 extern void mach_kpm_init(void); 96 97 /* 98 * External Data: 99 */ 100 extern int vac_size; /* cache size in bytes */ 101 extern uint_t vac_mask; /* VAC alignment consistency mask */ 102 extern uint_t vac_colors; 103 104 /* 105 * Global Data Definitions: 106 */ 107 108 /* 109 * XXX - Don't port this to new architectures 110 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 111 * 'romp' has no use with a prom with an IEEE 1275 client interface. 112 * The driver doesn't use the value, but it depends on the symbol. 113 */ 114 void *romp; /* veritas driver won't load without romp 4154976 */ 115 /* 116 * Declare these as initialized data so we can patch them. 117 */ 118 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 119 pgcnt_t segkpsize = 120 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 121 uint_t segmap_percent = 12; /* Size of segmap segment */ 122 123 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 124 int vac_copyback = 1; 125 char *cache_mode = NULL; 126 int use_mix = 1; 127 int prom_debug = 0; 128 129 struct bootops *bootops = 0; /* passed in from boot in %o2 */ 130 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 131 uint_t tba_taken_over = 0; 132 133 caddr_t s_text; /* start of kernel text segment */ 134 caddr_t e_text; /* end of kernel text segment */ 135 caddr_t s_data; /* start of kernel data segment */ 136 caddr_t e_data; /* end of kernel data segment */ 137 138 caddr_t modtext; /* beginning of module text */ 139 size_t modtext_sz; /* size of module text */ 140 caddr_t moddata; /* beginning of module data reserve */ 141 caddr_t e_moddata; /* end of module data reserve */ 142 143 /* 144 * End of first block of contiguous kernel in 32-bit virtual address space 145 */ 146 caddr_t econtig32; /* end of first blk of contiguous kernel */ 147 148 caddr_t ncbase; /* beginning of non-cached segment */ 149 caddr_t ncend; /* end of non-cached segment */ 150 caddr_t sdata; /* beginning of data segment */ 151 152 caddr_t extra_etva; /* beginning of unused nucleus text */ 153 pgcnt_t extra_etpg; /* number of pages of unused nucleus text */ 154 155 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 156 caddr_t nalloc_base; /* beginning of nucleus allocation */ 157 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 158 caddr_t valloc_base; /* beginning of kvalloc segment */ 159 160 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 161 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 162 163 uintptr_t shm_alignment; /* VAC address consistency modulus */ 164 struct memlist *phys_install; /* Total installed physical memory */ 165 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 166 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 167 struct memlist ndata; /* memlist of nucleus allocatable memory */ 168 int memexp_flag; /* memory expansion card flag */ 169 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 170 pgcnt_t obp_pages; /* Physical pages used by OBP */ 171 172 /* 173 * VM data structures 174 */ 175 long page_hashsz; /* Size of page hash table (power of two) */ 176 struct page *pp_base; /* Base of system page struct array */ 177 size_t pp_sz; /* Size in bytes of page struct array */ 178 struct page **page_hash; /* Page hash table */ 179 struct seg ktextseg; /* Segment used for kernel executable image */ 180 struct seg kvalloc; /* Segment used for "valloc" mapping */ 181 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 182 struct seg ktexthole; /* Segment used for nucleus text hole */ 183 struct seg kmapseg; /* Segment used for generic kernel mappings */ 184 struct seg kpmseg; /* Segment used for physical mapping */ 185 struct seg kdebugseg; /* Segment used for the kernel debugger */ 186 187 uintptr_t kpm_pp_base; /* Base of system kpm_page array */ 188 size_t kpm_pp_sz; /* Size of system kpm_page array */ 189 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 190 191 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 192 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 193 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 194 195 int segzio_fromheap = 0; /* zio allocations occur from heap */ 196 caddr_t segzio_base; /* Base address of segzio */ 197 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 198 199 /* 200 * debugger pages (if allocated) 201 */ 202 struct vnode kdebugvp; 203 204 /* 205 * VA range available to the debugger 206 */ 207 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 208 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 209 210 /* 211 * Segment for relocated kernel structures in 64-bit large RAM kernels 212 */ 213 struct seg kmem64; 214 215 struct memseg *memseg_base; 216 size_t memseg_sz; /* Used to translate a va to page */ 217 struct vnode unused_pages_vp; 218 219 /* 220 * VM data structures allocated early during boot. 221 */ 222 size_t pagehash_sz; 223 uint64_t memlist_sz; 224 225 char tbr_wr_addr_inited = 0; 226 227 228 /* 229 * Static Routines: 230 */ 231 static void memlist_add(uint64_t, uint64_t, struct memlist **, 232 struct memlist **); 233 static void kphysm_init(page_t *, struct memseg *, pgcnt_t, uintptr_t, 234 pgcnt_t); 235 static void kvm_init(void); 236 237 static void startup_init(void); 238 static void startup_memlist(void); 239 static void startup_modules(void); 240 static void startup_bop_gone(void); 241 static void startup_vm(void); 242 static void startup_end(void); 243 static void setup_cage_params(void); 244 static void startup_create_io_node(void); 245 246 static pgcnt_t npages; 247 static struct memlist *memlist; 248 void *memlist_end; 249 250 static pgcnt_t bop_alloc_pages; 251 static caddr_t hblk_base; 252 uint_t hblk_alloc_dynamic = 0; 253 uint_t hblk1_min = H1MIN; 254 uint_t hblk8_min; 255 256 257 /* 258 * Hooks for unsupported platforms and down-rev firmware 259 */ 260 int iam_positron(void); 261 #pragma weak iam_positron 262 static void do_prom_version_check(void); 263 static void kpm_init(void); 264 static void kpm_npages_setup(int); 265 static void kpm_memseg_init(void); 266 267 /* 268 * After receiving a thermal interrupt, this is the number of seconds 269 * to delay before shutting off the system, assuming 270 * shutdown fails. Use /etc/system to change the delay if this isn't 271 * large enough. 272 */ 273 int thermal_powerdown_delay = 1200; 274 275 /* 276 * Used to hold off page relocations into the cage until OBP has completed 277 * its boot-time handoff of its resources to the kernel. 278 */ 279 int page_relocate_ready = 0; 280 281 /* 282 * Enable some debugging messages concerning memory usage... 283 */ 284 #ifdef DEBUGGING_MEM 285 static int debugging_mem; 286 static void 287 printmemlist(char *title, struct memlist *list) 288 { 289 if (!debugging_mem) 290 return; 291 292 printf("%s\n", title); 293 294 while (list) { 295 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 296 (uint32_t)(list->address >> 32), (uint32_t)list->address, 297 (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 298 list = list->next; 299 } 300 } 301 302 void 303 printmemseg(struct memseg *memseg) 304 { 305 if (!debugging_mem) 306 return; 307 308 printf("memseg\n"); 309 310 while (memseg) { 311 prom_printf("\tpage = 0x%p, epage = 0x%p, " 312 "pfn = 0x%x, epfn = 0x%x\n", 313 memseg->pages, memseg->epages, 314 memseg->pages_base, memseg->pages_end); 315 memseg = memseg->next; 316 } 317 } 318 319 #define debug_pause(str) halt((str)) 320 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 321 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 322 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 323 #define MPRINTF3(str, a, b, c) \ 324 if (debugging_mem) prom_printf((str), (a), (b), (c)) 325 #else /* DEBUGGING_MEM */ 326 #define MPRINTF(str) 327 #define MPRINTF1(str, a) 328 #define MPRINTF2(str, a, b) 329 #define MPRINTF3(str, a, b, c) 330 #endif /* DEBUGGING_MEM */ 331 332 /* Simple message to indicate that the bootops pointer has been zeroed */ 333 #ifdef DEBUG 334 static int bootops_gone_on = 0; 335 #define BOOTOPS_GONE() \ 336 if (bootops_gone_on) \ 337 prom_printf("The bootops vec is zeroed now!\n"); 338 #else 339 #define BOOTOPS_GONE() 340 #endif /* DEBUG */ 341 342 /* 343 * Monitor pages may not be where this says they are. 344 * and the debugger may not be there either. 345 * 346 * Note that 'pages' here are *physical* pages, which are 8k on sun4u. 347 * 348 * Physical memory layout 349 * (not necessarily contiguous) 350 * (THIS IS SOMEWHAT WRONG) 351 * /-----------------------\ 352 * | monitor pages | 353 * availmem -|-----------------------| 354 * | | 355 * | page pool | 356 * | | 357 * |-----------------------| 358 * | configured tables | 359 * | buffers | 360 * firstaddr -|-----------------------| 361 * | hat data structures | 362 * |-----------------------| 363 * | kernel data, bss | 364 * |-----------------------| 365 * | interrupt stack | 366 * |-----------------------| 367 * | kernel text (RO) | 368 * |-----------------------| 369 * | trap table (4k) | 370 * |-----------------------| 371 * page 1 | panicbuf | 372 * |-----------------------| 373 * page 0 | reclaimed | 374 * |_______________________| 375 * 376 * 377 * 378 * Kernel's Virtual Memory Layout. 379 * /-----------------------\ 380 * 0xFFFFFFFF.FFFFFFFF -| |- 381 * | OBP's virtual page | 382 * | tables | 383 * 0xFFFFFFFC.00000000 -|-----------------------|- 384 * : : 385 * : : 386 * -|-----------------------|- 387 * | segzio | (base and size vary) 388 * 0xFFFFFE00.00000000 -|-----------------------|- 389 * | | Ultrasparc I/II support 390 * | segkpm segment | up to 2TB of physical 391 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 392 * | | 393 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 394 * : : 395 * : : 396 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 397 * | | ^ 398 * | UltraSPARC I/II call | | 399 * | bug requires an extra | | 400 * | 4 GB of space between | | 401 * | hole and used RAM | | 402 * | | | 403 * 0xFFFFF800.00000000 -|-----------------------|- | 404 * | | | 405 * | Virtual Address Hole | UltraSPARC 406 * | on UltraSPARC I/II | I/II * ONLY * 407 * | | | 408 * 0x00000800.00000000 -|-----------------------|- | 409 * | | | 410 * | UltraSPARC I/II call | | 411 * | bug requires an extra | | 412 * | 4 GB of space between | | 413 * | hole and used RAM | | 414 * | | v 415 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 416 * : : ^ 417 * : : | 418 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 419 * | | | 420 * | 64-bit kernel ONLY | | 421 * | | | 422 * | kmem64 segment | | 423 * | | | 424 * | (Relocated extra HME | Approximately 425 * | block allocations, | 1 TB of virtual 426 * | memnode freelists, | address space 427 * | HME hash buckets, | | 428 * | mml_table, kpmp_table,| | 429 * | page_t array and | | 430 * | hashblock pool to | | 431 * | avoid hard-coded | | 432 * | 32-bit vaddr | | 433 * | limitations) | | 434 * | | v 435 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 436 * | | 437 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 438 * | | 439 * 0x00000300.00000000 -|-----------------------|- SYSBASE 440 * : : 441 * : : 442 * -|-----------------------|- 443 * | | 444 * | segmap segment | SEGMAPSIZE (1/8th physmem, 445 * | | 256G MAX) 446 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 447 * : : 448 * : : 449 * -|-----------------------|- 450 * | | 451 * | segkp | SEGKPSIZE (2GB) 452 * | | 453 * | | 454 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 455 * | | 456 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 457 * | | (SEGKPBASE - 0x400000) 458 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 459 * | | (MEMSCRUBBASE - NCARGS) 460 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 461 * | | (ARGSBASE - PPMAPSIZE) 462 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 463 * | | 464 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 465 * | | 466 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 467 * : : 468 * : : 469 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 470 * | | 471 * | OBP | 472 * | | 473 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 474 * | kmdb | 475 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 476 * : : 477 * : : 478 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 479 * | | 480 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 481 * | | ~64MB) 482 * 0x00000000.78002000 -|-----------------------| 483 * | panicbuf | 484 * 0x00000000.78000000 -|-----------------------|- SYSBASE32 485 * : : 486 * : : 487 * | | 488 * |-----------------------|- econtig32 489 * | vm structures | 490 * 0x00000000.01C00000 |-----------------------|- nalloc_end 491 * | TSBs | 492 * |-----------------------|- end/nalloc_base 493 * | kernel data & bss | 494 * 0x00000000.01800000 -|-----------------------| 495 * : nucleus text hole : 496 * 0x00000000.01400000 -|-----------------------| 497 * : : 498 * |-----------------------| 499 * | module text | 500 * |-----------------------|- e_text/modtext 501 * | kernel text | 502 * |-----------------------| 503 * | trap table (48k) | 504 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 505 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 506 * |-----------------------| 507 * | | 508 * | invalid | 509 * | | 510 * 0x00000000.00000000 _|_______________________| 511 * 512 * 513 * 514 * 32-bit User Virtual Memory Layout. 515 * /-----------------------\ 516 * | | 517 * | invalid | 518 * | | 519 * 0xFFC00000 -|-----------------------|- USERLIMIT 520 * | user stack | 521 * : : 522 * : : 523 * : : 524 * | user data | 525 * -|-----------------------|- 526 * | user text | 527 * 0x00002000 -|-----------------------|- 528 * | invalid | 529 * 0x00000000 _|_______________________| 530 * 531 * 532 * 533 * 64-bit User Virtual Memory Layout. 534 * /-----------------------\ 535 * | | 536 * | invalid | 537 * | | 538 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 539 * | user stack | 540 * : : 541 * : : 542 * : : 543 * | user data | 544 * -|-----------------------|- 545 * | user text | 546 * 0x00000000.00100000 -|-----------------------|- 547 * | invalid | 548 * 0x00000000.00000000 _|_______________________| 549 */ 550 551 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 552 extern uint64_t ecache_flush_address(void); 553 554 #pragma weak load_platform_modules 555 #pragma weak plat_startup_memlist 556 #pragma weak ecache_init_scrub_flush_area 557 #pragma weak ecache_flush_address 558 559 560 /* 561 * By default the DR Cage is enabled for maximum OS 562 * MPSS performance. Users needing to disable the cage mechanism 563 * can set this variable to zero via /etc/system. 564 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 565 * will result in loss of DR functionality. 566 * Platforms wishing to disable kernel Cage by default 567 * should do so in their set_platform_defaults() routine. 568 */ 569 int kernel_cage_enable = 1; 570 571 static void 572 setup_cage_params(void) 573 { 574 void (*func)(void); 575 576 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 577 if (func != NULL) { 578 (*func)(); 579 return; 580 } 581 582 if (kernel_cage_enable == 0) { 583 return; 584 } 585 kcage_range_lock(); 586 if (kcage_range_init(phys_avail, 1) == 0) { 587 kcage_init(total_pages / 256); 588 } 589 kcage_range_unlock(); 590 591 if (kcage_on) { 592 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 593 } else { 594 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 595 } 596 597 } 598 599 /* 600 * Machine-dependent startup code 601 */ 602 void 603 startup(void) 604 { 605 startup_init(); 606 if (&startup_platform) 607 startup_platform(); 608 startup_memlist(); 609 startup_modules(); 610 setup_cage_params(); 611 startup_bop_gone(); 612 startup_vm(); 613 startup_end(); 614 } 615 616 struct regs sync_reg_buf; 617 uint64_t sync_tt; 618 619 void 620 sync_handler(void) 621 { 622 struct trap_info ti; 623 int i; 624 625 /* 626 * Prevent trying to talk to the other CPUs since they are 627 * sitting in the prom and won't reply. 628 */ 629 for (i = 0; i < NCPU; i++) { 630 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 631 cpu[i]->cpu_flags &= ~CPU_READY; 632 cpu[i]->cpu_flags |= CPU_QUIESCED; 633 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 634 } 635 } 636 637 /* 638 * We've managed to get here without going through the 639 * normal panic code path. Try and save some useful 640 * information. 641 */ 642 if (!panicstr && (curthread->t_panic_trap == NULL)) { 643 ti.trap_type = sync_tt; 644 ti.trap_regs = &sync_reg_buf; 645 ti.trap_addr = NULL; 646 ti.trap_mmu_fsr = 0x0; 647 648 curthread->t_panic_trap = &ti; 649 } 650 651 /* 652 * If we're re-entering the panic path, update the signature 653 * block so that the SC knows we're in the second part of panic. 654 */ 655 if (panicstr) 656 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 657 658 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 659 panic("sync initiated"); 660 } 661 662 663 static void 664 startup_init(void) 665 { 666 /* 667 * We want to save the registers while we're still in OBP 668 * so that we know they haven't been fiddled with since. 669 * (In principle, OBP can't change them just because it 670 * makes a callback, but we'd rather not depend on that 671 * behavior.) 672 */ 673 char sync_str[] = 674 "warning @ warning off : sync " 675 "%%tl-c %%tstate h# %p x! " 676 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 677 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 678 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 679 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 680 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 681 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 682 "%%y h# %p l! %%tl-c %%tt h# %p x! " 683 "sync ; warning !"; 684 685 /* 686 * 20 == num of %p substrings 687 * 16 == max num of chars %p will expand to. 688 */ 689 char bp[sizeof (sync_str) + 16 * 20]; 690 691 (void) check_boot_version(BOP_GETVERSION(bootops)); 692 693 /* 694 * Initialize ptl1 stack for the 1st CPU. 695 */ 696 ptl1_init_cpu(&cpu0); 697 698 /* 699 * Initialize the address map for cache consistent mappings 700 * to random pages; must be done after vac_size is set. 701 */ 702 ppmapinit(); 703 704 /* 705 * Initialize the PROM callback handler. 706 */ 707 init_vx_handler(); 708 709 /* 710 * have prom call sync_callback() to handle the sync and 711 * save some useful information which will be stored in the 712 * core file later. 713 */ 714 (void) sprintf((char *)bp, sync_str, 715 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 716 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 717 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 718 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 719 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 720 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 721 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 722 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 723 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 724 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 725 prom_interpret(bp, 0, 0, 0, 0, 0); 726 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 727 } 728 729 static u_longlong_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 730 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 731 732 #define IVSIZE ((MAXIVNUM * sizeof (intr_vec_t *)) + \ 733 (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 734 (MAX_RSVD_IVX * sizeof (intr_vecx_t))) 735 736 /* 737 * As OBP takes up some RAM when the system boots, pages will already be "lost" 738 * to the system and reflected in npages by the time we see it. 739 * 740 * We only want to allocate kernel structures in the 64-bit virtual address 741 * space on systems with enough RAM to make the overhead of keeping track of 742 * an extra kernel memory segment worthwhile. 743 * 744 * Since OBP has already performed its memory allocations by this point, if we 745 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 746 * memory in the 64-bit virtual address space; otherwise keep allocations 747 * contiguous with we've mapped so far in the 32-bit virtual address space. 748 */ 749 #define MINMOVE_RAM_MB ((size_t)1900) 750 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 751 752 pgcnt_t tune_npages = (pgcnt_t) 753 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 754 755 #pragma weak page_set_colorequiv_arr_cpu 756 extern void page_set_colorequiv_arr_cpu(void); 757 758 static void 759 startup_memlist(void) 760 { 761 size_t alloc_sz; 762 size_t ctrs_sz; 763 caddr_t alloc_base; 764 caddr_t ctrs_base, ctrs_end; 765 caddr_t memspace; 766 caddr_t va; 767 int memblocks = 0; 768 struct memlist *cur; 769 size_t syslimit = (size_t)SYSLIMIT; 770 size_t sysbase = (size_t)SYSBASE; 771 int alloc_alignsize = MMU_PAGESIZE; 772 extern void page_coloring_init(void); 773 extern void page_set_colorequiv_arr(void); 774 775 /* 776 * Initialize enough of the system to allow kmem_alloc to work by 777 * calling boot to allocate its memory until the time that 778 * kvm_init is completed. The page structs are allocated after 779 * rounding up end to the nearest page boundary; the memsegs are 780 * initialized and the space they use comes from the kernel heap. 781 * With appropriate initialization, they can be reallocated later 782 * to a size appropriate for the machine's configuration. 783 * 784 * At this point, memory is allocated for things that will never 785 * need to be freed, this used to be "valloced". This allows a 786 * savings as the pages don't need page structures to describe 787 * them because them will not be managed by the vm system. 788 */ 789 790 /* 791 * We're loaded by boot with the following configuration (as 792 * specified in the sun4u/conf/Mapfile): 793 * 794 * text: 4 MB chunk aligned on a 4MB boundary 795 * data & bss: 4 MB chunk aligned on a 4MB boundary 796 * 797 * These two chunks will eventually be mapped by 2 locked 4MB 798 * ttes and will represent the nucleus of the kernel. This gives 799 * us some free space that is already allocated, some or all of 800 * which is made available to kernel module text. 801 * 802 * The free space in the data-bss chunk is used for nucleus 803 * allocatable data structures and we reserve it using the 804 * nalloc_base and nalloc_end variables. This space is currently 805 * being used for hat data structures required for tlb miss 806 * handling operations. We align nalloc_base to a l2 cache 807 * linesize because this is the line size the hardware uses to 808 * maintain cache coherency. 809 * 256K is carved out for module data. 810 */ 811 812 nalloc_base = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 813 moddata = nalloc_base; 814 e_moddata = nalloc_base + MODDATA; 815 nalloc_base = e_moddata; 816 817 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 818 valloc_base = nalloc_base; 819 820 /* 821 * Calculate the start of the data segment. 822 */ 823 sdata = (caddr_t)((uintptr_t)e_data & MMU_PAGEMASK4M); 824 825 PRM_DEBUG(moddata); 826 PRM_DEBUG(nalloc_base); 827 PRM_DEBUG(nalloc_end); 828 PRM_DEBUG(sdata); 829 830 /* 831 * Remember any slop after e_text so we can give it to the modules. 832 */ 833 PRM_DEBUG(e_text); 834 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 835 if (((uintptr_t)modtext & MMU_PAGEMASK4M) != (uintptr_t)s_text) 836 panic("nucleus text overflow"); 837 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 838 modtext; 839 PRM_DEBUG(modtext); 840 PRM_DEBUG(modtext_sz); 841 842 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 843 &boot_physavail, &boot_physavail_len, 844 &boot_virtavail, &boot_virtavail_len); 845 /* 846 * Remember what the physically available highest page is 847 * so that dumpsys works properly, and find out how much 848 * memory is installed. 849 */ 850 installed_top_size_memlist_array(boot_physinstalled, 851 boot_physinstalled_len, &physmax, &physinstalled); 852 PRM_DEBUG(physinstalled); 853 PRM_DEBUG(physmax); 854 855 /* Fill out memory nodes config structure */ 856 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 857 858 /* 859 * Get the list of physically available memory to size 860 * the number of page structures needed. 861 */ 862 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 863 /* 864 * This first snap shot of npages can represent the pages used 865 * by OBP's text and data approximately. This is used in the 866 * the calculation of the kernel size 867 */ 868 obp_pages = physinstalled - npages; 869 870 871 /* 872 * On small-memory systems (<MODTEXT_SM_SIZE MB, currently 256MB), the 873 * in-nucleus module text is capped to MODTEXT_SM_CAP bytes (currently 874 * 2MB) and any excess pages are put on physavail. The assumption is 875 * that small-memory systems will need more pages more than they'll 876 * need efficiently-mapped module texts. 877 */ 878 if ((physinstalled < mmu_btop(MODTEXT_SM_SIZE << 20)) && 879 modtext_sz > MODTEXT_SM_CAP) { 880 extra_etpg = mmu_btop(modtext_sz - MODTEXT_SM_CAP); 881 modtext_sz = MODTEXT_SM_CAP; 882 } else 883 extra_etpg = 0; 884 PRM_DEBUG(extra_etpg); 885 PRM_DEBUG(modtext_sz); 886 extra_etva = modtext + modtext_sz; 887 PRM_DEBUG(extra_etva); 888 889 /* 890 * Account for any pages after e_text and e_data. 891 */ 892 npages += extra_etpg; 893 npages += mmu_btopr(nalloc_end - nalloc_base); 894 PRM_DEBUG(npages); 895 896 /* 897 * npages is the maximum of available physical memory possible. 898 * (ie. it will never be more than this) 899 */ 900 901 /* 902 * initialize the nucleus memory allocator. 903 */ 904 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 905 906 /* 907 * Allocate mmu fault status area from the nucleus data area. 908 */ 909 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 910 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 911 912 /* 913 * Allocate kernel TSBs from the nucleus data area. 914 */ 915 if (ndata_alloc_tsbs(&ndata, npages) != 0) 916 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 917 918 /* 919 * Allocate dmv dispatch table from the nucleus data area. 920 */ 921 if (ndata_alloc_dmv(&ndata) != 0) 922 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 923 924 925 page_coloring_init(); 926 927 /* 928 * Allocate page_freelists bin headers for memnode 0 from the 929 * nucleus data area. 930 */ 931 if (ndata_alloc_page_freelists(&ndata, 0) != 0) 932 cmn_err(CE_PANIC, 933 "no more nucleus memory after page free lists alloc"); 934 935 if (kpm_enable) { 936 kpm_init(); 937 /* 938 * kpm page space -- Update kpm_npages and make the 939 * same assumption about fragmenting as it is done 940 * for memseg_sz. 941 */ 942 kpm_npages_setup(memblocks + 4); 943 } 944 945 /* 946 * Allocate hat related structs from the nucleus data area. 947 */ 948 if (ndata_alloc_hat(&ndata, npages, kpm_npages) != 0) 949 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 950 951 /* 952 * We want to do the BOP_ALLOCs before the real allocation of page 953 * structs in order to not have to allocate page structs for this 954 * memory. We need to calculate a virtual address because we want 955 * the page structs to come before other allocations in virtual address 956 * space. This is so some (if not all) of page structs can actually 957 * live in the nucleus. 958 */ 959 960 /* 961 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 962 * 963 * There are comments all over the SFMMU code warning of dire 964 * consequences if the TSBs are moved out of 32-bit space. This 965 * is largely because the asm code uses "sethi %hi(addr)"-type 966 * instructions which will not provide the expected result if the 967 * address is a 64-bit one. 968 * 969 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 970 */ 971 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 972 alloc_base = sfmmu_ktsb_alloc(alloc_base); 973 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 974 PRM_DEBUG(alloc_base); 975 976 /* 977 * Allocate IOMMU TSB array. We do this here so that the physical 978 * memory gets deducted from the PROM's physical memory list. 979 */ 980 alloc_base = iommu_tsb_init(alloc_base); 981 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 982 ecache_alignsize); 983 PRM_DEBUG(alloc_base); 984 985 /* 986 * Platforms like Starcat and OPL need special structures assigned in 987 * 32-bit virtual address space because their probing routines execute 988 * FCode, and FCode can't handle 64-bit virtual addresses... 989 */ 990 if (&plat_startup_memlist) { 991 alloc_base = plat_startup_memlist(alloc_base); 992 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 993 ecache_alignsize); 994 PRM_DEBUG(alloc_base); 995 } 996 997 /* 998 * If we have enough memory, use 4M pages for alignment because it 999 * greatly reduces the number of TLB misses we take albeit at the cost 1000 * of possible RAM wastage (degenerate case of 4 MB - MMU_PAGESIZE per 1001 * allocation.) Still, the speedup on large memory systems (e.g. > 64 1002 * GB) is quite noticeable, so it is worth the effort to do if we can. 1003 * 1004 * Note, however, that this speedup will only occur if the boot PROM 1005 * uses the largest possible MMU page size possible to map memory 1006 * requests that are properly aligned and sized (for example, a request 1007 * for a multiple of 4MB of memory aligned to a 4MB boundary will 1008 * result in a mapping using a 4MB MMU page.) 1009 * 1010 * Even then, the large page mappings will only speed things up until 1011 * the startup process proceeds a bit further, as when 1012 * sfmmu_map_prom_mappings() copies page mappings from the PROM to the 1013 * kernel it remaps everything but the TSBs using 8K pages anyway... 1014 * 1015 * At some point in the future, sfmmu_map_prom_mappings() will be 1016 * rewritten to copy memory mappings to the kernel using the same MMU 1017 * page sizes the PROM used. When that occurs, if the PROM did use 1018 * large MMU pages to map memory, the alignment/sizing work we're 1019 * doing now should give us a nice extra performance boost, albeit at 1020 * the cost of greater RAM usage... 1021 */ 1022 alloc_alignsize = ((npages >= tune_npages) ? MMU_PAGESIZE4M : 1023 MMU_PAGESIZE); 1024 1025 PRM_DEBUG(tune_npages); 1026 PRM_DEBUG(alloc_alignsize); 1027 1028 /* 1029 * Save off where the contiguous allocations to date have ended 1030 * in econtig32. 1031 */ 1032 econtig32 = alloc_base; 1033 PRM_DEBUG(econtig32); 1034 1035 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1036 cmn_err(CE_PANIC, "econtig32 too big"); 1037 1038 /* 1039 * To avoid memory allocation collisions in the 32-bit virtual address 1040 * space, make allocations from this point forward in 64-bit virtual 1041 * address space starting at syslimit and working up. Also use the 1042 * alignment specified by alloc_alignsize, as we may be able to save 1043 * ourselves TLB misses by using larger page sizes if they're 1044 * available. 1045 * 1046 * All this is needed because on large memory systems, the default 1047 * Solaris allocations will collide with SYSBASE32, which is hard 1048 * coded to be at the virtual address 0x78000000. Therefore, on 64-bit 1049 * kernels, move the allocations to a location in the 64-bit virtual 1050 * address space space, allowing those structures to grow without 1051 * worry. 1052 * 1053 * On current CPUs we'll run out of physical memory address bits before 1054 * we need to worry about the allocations running into anything else in 1055 * VM or the virtual address holes on US-I and II, as there's currently 1056 * about 1 TB of addressable space before the US-I/II VA hole. 1057 */ 1058 kmem64_base = (caddr_t)syslimit; 1059 PRM_DEBUG(kmem64_base); 1060 1061 alloc_base = (caddr_t)roundup((uintptr_t)kmem64_base, alloc_alignsize); 1062 1063 /* 1064 * If KHME and/or UHME hash buckets won't fit in the nucleus, allocate 1065 * them here. 1066 */ 1067 if (khme_hash == NULL || uhme_hash == NULL) { 1068 /* 1069 * alloc_hme_buckets() will align alloc_base properly before 1070 * assigning the hash buckets, so we don't need to do it 1071 * before the call... 1072 */ 1073 alloc_base = alloc_hme_buckets(alloc_base, alloc_alignsize); 1074 1075 PRM_DEBUG(alloc_base); 1076 PRM_DEBUG(khme_hash); 1077 PRM_DEBUG(uhme_hash); 1078 } 1079 1080 /* 1081 * Allocate the remaining page freelists. NUMA systems can 1082 * have lots of page freelists, one per node, which quickly 1083 * outgrow the amount of nucleus memory available. 1084 */ 1085 if (max_mem_nodes > 1) { 1086 int mnode; 1087 caddr_t alloc_start = alloc_base; 1088 1089 for (mnode = 1; mnode < max_mem_nodes; mnode++) { 1090 alloc_base = alloc_page_freelists(mnode, alloc_base, 1091 ecache_alignsize); 1092 } 1093 1094 if (alloc_base > alloc_start) { 1095 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1096 alloc_alignsize); 1097 if ((caddr_t)BOP_ALLOC(bootops, alloc_start, 1098 alloc_base - alloc_start, 1099 alloc_alignsize) != alloc_start) 1100 cmn_err(CE_PANIC, 1101 "Unable to alloc page freelists\n"); 1102 } 1103 1104 PRM_DEBUG(alloc_base); 1105 } 1106 1107 if (!mml_table) { 1108 size_t mmltable_sz; 1109 1110 /* 1111 * We need to allocate the mml_table here because there 1112 * was not enough space within the nucleus. 1113 */ 1114 mmltable_sz = sizeof (kmutex_t) * mml_table_sz; 1115 alloc_sz = roundup(mmltable_sz, alloc_alignsize); 1116 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1117 alloc_alignsize); 1118 1119 if ((mml_table = (kmutex_t *)BOP_ALLOC(bootops, alloc_base, 1120 alloc_sz, alloc_alignsize)) != (kmutex_t *)alloc_base) 1121 panic("mml_table alloc failure"); 1122 1123 alloc_base += alloc_sz; 1124 PRM_DEBUG(mml_table); 1125 PRM_DEBUG(alloc_base); 1126 } 1127 1128 if (kpm_enable && !(kpmp_table || kpmp_stable)) { 1129 size_t kpmptable_sz; 1130 caddr_t table; 1131 1132 /* 1133 * We need to allocate either kpmp_table or kpmp_stable here 1134 * because there was not enough space within the nucleus. 1135 */ 1136 kpmptable_sz = (kpm_smallpages == 0) ? 1137 sizeof (kpm_hlk_t) * kpmp_table_sz : 1138 sizeof (kpm_shlk_t) * kpmp_stable_sz; 1139 1140 alloc_sz = roundup(kpmptable_sz, alloc_alignsize); 1141 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1142 alloc_alignsize); 1143 1144 table = BOP_ALLOC(bootops, alloc_base, alloc_sz, 1145 alloc_alignsize); 1146 1147 if (table != alloc_base) 1148 panic("kpmp_table or kpmp_stable alloc failure"); 1149 1150 if (kpm_smallpages == 0) { 1151 kpmp_table = (kpm_hlk_t *)table; 1152 PRM_DEBUG(kpmp_table); 1153 } else { 1154 kpmp_stable = (kpm_shlk_t *)table; 1155 PRM_DEBUG(kpmp_stable); 1156 } 1157 1158 alloc_base += alloc_sz; 1159 PRM_DEBUG(alloc_base); 1160 } 1161 1162 if (&ecache_init_scrub_flush_area) { 1163 /* 1164 * Pass alloc_base directly, as the routine itself is 1165 * responsible for any special alignment requirements... 1166 */ 1167 alloc_base = ecache_init_scrub_flush_area(alloc_base); 1168 PRM_DEBUG(alloc_base); 1169 } 1170 1171 /* 1172 * Take the most current snapshot we can by calling mem-update. 1173 */ 1174 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1175 &boot_physavail, &boot_physavail_len, 1176 &boot_virtavail, &boot_virtavail_len); 1177 1178 /* 1179 * Reset npages and memblocks based on boot_physavail list. 1180 */ 1181 size_physavail(boot_physavail, boot_physavail_len, &npages, &memblocks); 1182 PRM_DEBUG(npages); 1183 1184 /* 1185 * Account for extra memory after e_text. 1186 */ 1187 npages += extra_etpg; 1188 1189 /* 1190 * Calculate the largest free memory chunk in the nucleus data area. 1191 * We need to figure out if page structs can fit in there or not. 1192 * We also make sure enough page structs get created for any physical 1193 * memory we might be returning to the system. 1194 */ 1195 ndata_remain_sz = ndata_maxsize(&ndata); 1196 PRM_DEBUG(ndata_remain_sz); 1197 1198 pp_sz = sizeof (struct page) * npages; 1199 1200 /* 1201 * Here's a nice bit of code based on somewhat recursive logic: 1202 * 1203 * If the page array would fit within the nucleus, we want to 1204 * add npages to cover any extra memory we may be returning back 1205 * to the system. 1206 * 1207 * HOWEVER, the page array is sized by calculating the size of 1208 * (struct page * npages), as are the pagehash table, ctrs and 1209 * memseg_list, so the very act of performing the calculation below may 1210 * in fact make the array large enough that it no longer fits in the 1211 * nucleus, meaning there would now be a much larger area of the 1212 * nucleus free that should really be added to npages, which would 1213 * make the page array that much larger, and so on. 1214 * 1215 * This also ignores the memory possibly used in the nucleus for the 1216 * the page hash, ctrs and memseg list and the fact that whether they 1217 * fit there or not varies with the npages calculation below, but we 1218 * don't even factor them into the equation at this point; perhaps we 1219 * should or perhaps we should just take the approach that the few 1220 * extra pages we could add via this calculation REALLY aren't worth 1221 * the hassle... 1222 */ 1223 if (ndata_remain_sz > pp_sz) { 1224 size_t spare = ndata_spare(&ndata, pp_sz, ecache_alignsize); 1225 1226 npages += mmu_btop(spare); 1227 1228 pp_sz = npages * sizeof (struct page); 1229 1230 pp_base = ndata_alloc(&ndata, pp_sz, ecache_alignsize); 1231 } 1232 1233 /* 1234 * If physmem is patched to be non-zero, use it instead of 1235 * the monitor value unless physmem is larger than the total 1236 * amount of memory on hand. 1237 */ 1238 if (physmem == 0 || physmem > npages) 1239 physmem = npages; 1240 1241 /* 1242 * If pp_base is NULL that means the routines above have determined 1243 * the page array will not fit in the nucleus; we'll have to 1244 * BOP_ALLOC() ourselves some space for them. 1245 */ 1246 if (pp_base == NULL) { 1247 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1248 alloc_alignsize); 1249 1250 alloc_sz = roundup(pp_sz, alloc_alignsize); 1251 1252 if ((pp_base = (struct page *)BOP_ALLOC(bootops, 1253 alloc_base, alloc_sz, alloc_alignsize)) != 1254 (struct page *)alloc_base) 1255 panic("page alloc failure"); 1256 1257 alloc_base += alloc_sz; 1258 } 1259 1260 /* 1261 * The page structure hash table size is a power of 2 1262 * such that the average hash chain length is PAGE_HASHAVELEN. 1263 */ 1264 page_hashsz = npages / PAGE_HASHAVELEN; 1265 page_hashsz = 1 << highbit((ulong_t)page_hashsz); 1266 pagehash_sz = sizeof (struct page *) * page_hashsz; 1267 1268 /* 1269 * We want to TRY to fit the page structure hash table, 1270 * the page size free list counters, the memseg list and 1271 * and the kpm page space in the nucleus if possible. 1272 * 1273 * alloc_sz counts how much memory needs to be allocated by 1274 * BOP_ALLOC(). 1275 */ 1276 page_hash = ndata_alloc(&ndata, pagehash_sz, ecache_alignsize); 1277 1278 alloc_sz = (page_hash == NULL ? pagehash_sz : 0); 1279 1280 /* 1281 * Size up per page size free list counters. 1282 */ 1283 ctrs_sz = page_ctrs_sz(); 1284 ctrs_base = ndata_alloc(&ndata, ctrs_sz, ecache_alignsize); 1285 1286 if (ctrs_base == NULL) 1287 alloc_sz = roundup(alloc_sz, ecache_alignsize) + ctrs_sz; 1288 1289 /* 1290 * The memseg list is for the chunks of physical memory that 1291 * will be managed by the vm system. The number calculated is 1292 * a guess as boot may fragment it more when memory allocations 1293 * are made before kphysm_init(). Currently, there are two 1294 * allocations before then, so we assume each causes fragmen- 1295 * tation, and add a couple more for good measure. 1296 */ 1297 memseg_sz = sizeof (struct memseg) * (memblocks + 4); 1298 memseg_base = ndata_alloc(&ndata, memseg_sz, ecache_alignsize); 1299 1300 if (memseg_base == NULL) 1301 alloc_sz = roundup(alloc_sz, ecache_alignsize) + memseg_sz; 1302 1303 1304 if (kpm_enable) { 1305 /* 1306 * kpm page space -- Update kpm_npages and make the 1307 * same assumption about fragmenting as it is done 1308 * for memseg_sz above. 1309 */ 1310 kpm_npages_setup(memblocks + 4); 1311 kpm_pp_sz = (kpm_smallpages == 0) ? 1312 kpm_npages * sizeof (kpm_page_t): 1313 kpm_npages * sizeof (kpm_spage_t); 1314 1315 kpm_pp_base = (uintptr_t)ndata_alloc(&ndata, kpm_pp_sz, 1316 ecache_alignsize); 1317 1318 if (kpm_pp_base == NULL) 1319 alloc_sz = roundup(alloc_sz, ecache_alignsize) + 1320 kpm_pp_sz; 1321 } 1322 1323 if (alloc_sz > 0) { 1324 uintptr_t bop_base; 1325 1326 /* 1327 * We need extra memory allocated through BOP_ALLOC. 1328 */ 1329 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1330 alloc_alignsize); 1331 1332 alloc_sz = roundup(alloc_sz, alloc_alignsize); 1333 1334 if ((bop_base = (uintptr_t)BOP_ALLOC(bootops, alloc_base, 1335 alloc_sz, alloc_alignsize)) != (uintptr_t)alloc_base) 1336 panic("system page struct alloc failure"); 1337 1338 alloc_base += alloc_sz; 1339 1340 if (page_hash == NULL) { 1341 page_hash = (struct page **)bop_base; 1342 bop_base = roundup(bop_base + pagehash_sz, 1343 ecache_alignsize); 1344 } 1345 1346 if (ctrs_base == NULL) { 1347 ctrs_base = (caddr_t)bop_base; 1348 bop_base = roundup(bop_base + ctrs_sz, 1349 ecache_alignsize); 1350 } 1351 1352 if (memseg_base == NULL) { 1353 memseg_base = (struct memseg *)bop_base; 1354 bop_base = roundup(bop_base + memseg_sz, 1355 ecache_alignsize); 1356 } 1357 1358 if (kpm_enable && kpm_pp_base == NULL) { 1359 kpm_pp_base = (uintptr_t)bop_base; 1360 bop_base = roundup(bop_base + kpm_pp_sz, 1361 ecache_alignsize); 1362 } 1363 1364 ASSERT(bop_base <= (uintptr_t)alloc_base); 1365 } 1366 1367 /* 1368 * Initialize per page size free list counters. 1369 */ 1370 ctrs_end = page_ctrs_alloc(ctrs_base); 1371 ASSERT(ctrs_base + ctrs_sz >= ctrs_end); 1372 1373 PRM_DEBUG(page_hash); 1374 PRM_DEBUG(memseg_base); 1375 PRM_DEBUG(kpm_pp_base); 1376 PRM_DEBUG(kpm_pp_sz); 1377 PRM_DEBUG(pp_base); 1378 PRM_DEBUG(pp_sz); 1379 PRM_DEBUG(alloc_base); 1380 1381 #ifdef TRAPTRACE 1382 /* 1383 * Allocate trap trace buffer last so as not to affect 1384 * the 4M alignments of the allocations above on V9 SPARCs... 1385 */ 1386 alloc_base = trap_trace_alloc(alloc_base); 1387 PRM_DEBUG(alloc_base); 1388 #endif /* TRAPTRACE */ 1389 1390 if (kmem64_base) { 1391 /* 1392 * Set the end of the kmem64 segment for V9 SPARCs, if 1393 * appropriate... 1394 */ 1395 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, 1396 alloc_alignsize); 1397 1398 PRM_DEBUG(kmem64_base); 1399 PRM_DEBUG(kmem64_end); 1400 } 1401 1402 /* 1403 * Allocate space for the interrupt vector table and also for the 1404 * reserved interrupt vector data structures. 1405 */ 1406 memspace = (caddr_t)BOP_ALLOC(bootops, (caddr_t)intr_vec_table, 1407 IVSIZE, MMU_PAGESIZE); 1408 if (memspace != (caddr_t)intr_vec_table) 1409 panic("interrupt vector table allocation failure"); 1410 1411 /* 1412 * The memory lists from boot are allocated from the heap arena 1413 * so that later they can be freed and/or reallocated. 1414 */ 1415 if (BOP_GETPROP(bootops, "extent", &memlist_sz) == -1) 1416 panic("could not retrieve property \"extent\""); 1417 1418 /* 1419 * Between now and when we finish copying in the memory lists, 1420 * allocations happen so the space gets fragmented and the 1421 * lists longer. Leave enough space for lists twice as long 1422 * as what boot says it has now; roundup to a pagesize. 1423 * Also add space for the final phys-avail copy in the fixup 1424 * routine. 1425 */ 1426 va = (caddr_t)(sysbase + PAGESIZE + PANICBUFSIZE + 1427 roundup(IVSIZE, MMU_PAGESIZE)); 1428 memlist_sz *= 4; 1429 memlist_sz = roundup(memlist_sz, MMU_PAGESIZE); 1430 memspace = (caddr_t)BOP_ALLOC(bootops, va, memlist_sz, BO_NO_ALIGN); 1431 if (memspace == NULL) 1432 halt("Boot allocation failed."); 1433 1434 memlist = (struct memlist *)memspace; 1435 memlist_end = (char *)memspace + memlist_sz; 1436 1437 PRM_DEBUG(memlist); 1438 PRM_DEBUG(memlist_end); 1439 PRM_DEBUG(sysbase); 1440 PRM_DEBUG(syslimit); 1441 1442 kernelheap_init((void *)sysbase, (void *)syslimit, 1443 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1444 1445 /* 1446 * Take the most current snapshot we can by calling mem-update. 1447 */ 1448 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1449 &boot_physavail, &boot_physavail_len, 1450 &boot_virtavail, &boot_virtavail_len); 1451 1452 /* 1453 * Remove the space used by BOP_ALLOC from the kernel heap 1454 * plus the area actually used by the OBP (if any) 1455 * ignoring virtual addresses in virt_avail, above syslimit. 1456 */ 1457 virt_avail = memlist; 1458 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1459 1460 for (cur = virt_avail; cur->next; cur = cur->next) { 1461 uint64_t range_base, range_size; 1462 1463 if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1464 continue; 1465 if (range_base >= (uint64_t)syslimit) 1466 break; 1467 /* 1468 * Limit the range to end at syslimit. 1469 */ 1470 range_size = MIN(cur->next->address, 1471 (uint64_t)syslimit) - range_base; 1472 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1473 0, 0, (void *)range_base, (void *)(range_base + range_size), 1474 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1475 } 1476 1477 phys_avail = memlist; 1478 (void) copy_physavail(boot_physavail, boot_physavail_len, 1479 &memlist, 0, 0); 1480 1481 /* 1482 * Add any extra memory after e_text to the phys_avail list, as long 1483 * as there's at least a page to add. 1484 */ 1485 if (extra_etpg) 1486 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1487 &memlist, &phys_avail); 1488 1489 /* 1490 * Add any extra memory after e_data to the phys_avail list as long 1491 * as there's at least a page to add. Usually, there isn't any, 1492 * since extra HME blocks typically get allocated there first before 1493 * using RAM elsewhere. 1494 */ 1495 if ((nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE)) == NULL) 1496 nalloc_base = nalloc_end; 1497 ndata_remain_sz = nalloc_end - nalloc_base; 1498 1499 if (ndata_remain_sz >= MMU_PAGESIZE) 1500 memlist_add(va_to_pa(nalloc_base), 1501 (uint64_t)ndata_remain_sz, &memlist, &phys_avail); 1502 1503 PRM_DEBUG(memlist); 1504 PRM_DEBUG(memlist_sz); 1505 PRM_DEBUG(memspace); 1506 1507 if ((caddr_t)memlist > (memspace + memlist_sz)) 1508 panic("memlist overflow"); 1509 1510 PRM_DEBUG(pp_base); 1511 PRM_DEBUG(memseg_base); 1512 PRM_DEBUG(npages); 1513 1514 /* 1515 * Initialize the page structures from the memory lists. 1516 */ 1517 kphysm_init(pp_base, memseg_base, npages, kpm_pp_base, kpm_npages); 1518 1519 availrmem_initial = availrmem = freemem; 1520 PRM_DEBUG(availrmem); 1521 1522 /* 1523 * Some of the locks depend on page_hashsz being set! 1524 * kmem_init() depends on this; so, keep it here. 1525 */ 1526 page_lock_init(); 1527 1528 /* 1529 * Initialize kernel memory allocator. 1530 */ 1531 kmem_init(); 1532 1533 /* 1534 * Factor in colorequiv to check additional 'equivalent' bins 1535 */ 1536 if (&page_set_colorequiv_arr_cpu != NULL) 1537 page_set_colorequiv_arr_cpu(); 1538 else 1539 page_set_colorequiv_arr(); 1540 1541 /* 1542 * Initialize bp_mapin(). 1543 */ 1544 bp_init(shm_alignment, HAT_STRICTORDER); 1545 1546 /* 1547 * Reserve space for panicbuf, intr_vec_table and reserved interrupt 1548 * vector data structures from the 32-bit heap. 1549 */ 1550 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1551 panicbuf, panicbuf + PANICBUFSIZE, 1552 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1553 1554 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1555 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1556 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1557 1558 mem_config_init(); 1559 } 1560 1561 static void 1562 startup_modules(void) 1563 { 1564 int proplen, nhblk1, nhblk8; 1565 size_t nhblksz; 1566 pgcnt_t hblk_pages, pages_per_hblk; 1567 size_t hme8blk_sz, hme1blk_sz; 1568 1569 /* 1570 * Log any optional messages from the boot program 1571 */ 1572 proplen = (size_t)BOP_GETPROPLEN(bootops, "boot-message"); 1573 if (proplen > 0) { 1574 char *msg; 1575 size_t len = (size_t)proplen; 1576 1577 msg = kmem_zalloc(len, KM_SLEEP); 1578 (void) BOP_GETPROP(bootops, "boot-message", msg); 1579 cmn_err(CE_CONT, "?%s\n", msg); 1580 kmem_free(msg, len); 1581 } 1582 1583 /* 1584 * Let the platforms have a chance to change default 1585 * values before reading system file. 1586 */ 1587 if (&set_platform_defaults) 1588 set_platform_defaults(); 1589 1590 /* 1591 * Calculate default settings of system parameters based upon 1592 * maxusers, yet allow to be overridden via the /etc/system file. 1593 */ 1594 param_calc(0); 1595 1596 mod_setup(); 1597 1598 /* 1599 * If this is a positron, complain and halt. 1600 */ 1601 if (&iam_positron && iam_positron()) { 1602 cmn_err(CE_WARN, "This hardware platform is not supported" 1603 " by this release of Solaris.\n"); 1604 #ifdef DEBUG 1605 prom_enter_mon(); /* Type 'go' to resume */ 1606 cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1607 cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1608 1609 #else /* DEBUG */ 1610 halt(0); 1611 #endif /* DEBUG */ 1612 } 1613 1614 /* 1615 * If we are running firmware that isn't 64-bit ready 1616 * then complain and halt. 1617 */ 1618 do_prom_version_check(); 1619 1620 /* 1621 * Initialize system parameters 1622 */ 1623 param_init(); 1624 1625 /* 1626 * maxmem is the amount of physical memory we're playing with. 1627 */ 1628 maxmem = physmem; 1629 1630 /* Set segkp limits. */ 1631 ncbase = kdi_segdebugbase; 1632 ncend = kdi_segdebugbase; 1633 1634 /* 1635 * Initialize the hat layer. 1636 */ 1637 hat_init(); 1638 1639 /* 1640 * Initialize segment management stuff. 1641 */ 1642 seg_init(); 1643 1644 /* 1645 * Create the va>tte handler, so the prom can understand 1646 * kernel translations. The handler is installed later, just 1647 * as we are about to take over the trap table from the prom. 1648 */ 1649 create_va_to_tte(); 1650 1651 /* 1652 * Load the forthdebugger (optional) 1653 */ 1654 forthdebug_init(); 1655 1656 /* 1657 * Create OBP node for console input callbacks 1658 * if it is needed. 1659 */ 1660 startup_create_io_node(); 1661 1662 if (modloadonly("fs", "specfs") == -1) 1663 halt("Can't load specfs"); 1664 1665 if (modloadonly("fs", "devfs") == -1) 1666 halt("Can't load devfs"); 1667 1668 if (modloadonly("misc", "swapgeneric") == -1) 1669 halt("Can't load swapgeneric"); 1670 1671 (void) modloadonly("sys", "lbl_edition"); 1672 1673 dispinit(); 1674 1675 /* 1676 * Infer meanings to the members of the idprom buffer. 1677 */ 1678 parse_idprom(); 1679 1680 /* Read cluster configuration data. */ 1681 clconf_init(); 1682 1683 setup_ddi(); 1684 1685 /* 1686 * Lets take this opportunity to load the root device. 1687 */ 1688 if (loadrootmodules() != 0) 1689 debug_enter("Can't load the root filesystem"); 1690 1691 /* 1692 * Load tod driver module for the tod part found on this system. 1693 * Recompute the cpu frequency/delays based on tod as tod part 1694 * tends to keep time more accurately. 1695 */ 1696 if (&load_tod_module) 1697 load_tod_module(); 1698 1699 /* 1700 * Allow platforms to load modules which might 1701 * be needed after bootops are gone. 1702 */ 1703 if (&load_platform_modules) 1704 load_platform_modules(); 1705 1706 setcpudelay(); 1707 1708 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1709 &boot_physavail, &boot_physavail_len, 1710 &boot_virtavail, &boot_virtavail_len); 1711 1712 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1713 1714 /* 1715 * Calculation and allocation of hmeblks needed to remap 1716 * the memory allocated by PROM till now: 1717 * 1718 * (1) calculate how much virtual memory has been bop_alloc'ed. 1719 * (2) roundup this memory to span of hme8blk, i.e. 64KB 1720 * (3) calculate number of hme8blk's needed to remap this memory 1721 * (4) calculate amount of memory that's consumed by these hme8blk's 1722 * (5) add memory calculated in steps (2) and (4) above. 1723 * (6) roundup this memory to span of hme8blk, i.e. 64KB 1724 * (7) calculate number of hme8blk's needed to remap this memory 1725 * (8) calculate amount of memory that's consumed by these hme8blk's 1726 * (9) allocate additional hme1blk's to hold large mappings. 1727 * H8TOH1 determines this. The current SWAG gives enough hblk1's 1728 * to remap everything with 4M mappings. 1729 * (10) account for partially used hblk8's due to non-64K aligned 1730 * PROM mapping entries. 1731 * (11) add memory calculated in steps (8), (9), and (10) above. 1732 * (12) kmem_zalloc the memory calculated in (11); since segkmem 1733 * is not ready yet, this gets bop_alloc'ed. 1734 * (13) there will be very few bop_alloc's after this point before 1735 * trap table takes over 1736 */ 1737 1738 /* sfmmu_init_nucleus_hblks expects properly aligned data structures. */ 1739 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1740 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1741 1742 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1743 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1744 nhblk8 = bop_alloc_pages / pages_per_hblk; 1745 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1746 hblk_pages = btopr(nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz); 1747 bop_alloc_pages += hblk_pages; 1748 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1749 nhblk8 = bop_alloc_pages / pages_per_hblk; 1750 nhblk1 = roundup(nhblk8, H8TOH1) / H8TOH1; 1751 if (nhblk1 < hblk1_min) 1752 nhblk1 = hblk1_min; 1753 if (nhblk8 < hblk8_min) 1754 nhblk8 = hblk8_min; 1755 1756 /* 1757 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1758 * boundary, the number of hblk8's needed to map the entries in the 1759 * boot_virtavail list needs to be adjusted to take this into 1760 * consideration. Thus, we need to add additional hblk8's since it 1761 * is possible that an hblk8 will not have all 8 slots used due to 1762 * alignment constraints. Since there were boot_virtavail_len entries 1763 * in that list, we need to add that many hblk8's to the number 1764 * already calculated to make sure we don't underestimate. 1765 */ 1766 nhblk8 += boot_virtavail_len; 1767 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1768 1769 /* Allocate in pagesize chunks */ 1770 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1771 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1772 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1773 } 1774 1775 static void 1776 startup_bop_gone(void) 1777 { 1778 extern int bop_io_quiesced; 1779 1780 /* 1781 * Destroy the MD initialized at startup 1782 * The startup initializes the MD framework 1783 * using prom and BOP alloc free it now. 1784 */ 1785 mach_descrip_startup_fini(); 1786 1787 /* 1788 * Call back into boot and release boots resources. 1789 */ 1790 BOP_QUIESCE_IO(bootops); 1791 bop_io_quiesced = 1; 1792 1793 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1794 &boot_physavail, &boot_physavail_len, 1795 &boot_virtavail, &boot_virtavail_len); 1796 /* 1797 * Copy physinstalled list into kernel space. 1798 */ 1799 phys_install = memlist; 1800 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1801 1802 /* 1803 * setup physically contiguous area twice as large as the ecache. 1804 * this is used while doing displacement flush of ecaches 1805 */ 1806 if (&ecache_flush_address) { 1807 ecache_flushaddr = ecache_flush_address(); 1808 if (ecache_flushaddr == (uint64_t)-1) { 1809 cmn_err(CE_PANIC, 1810 "startup: no memory to set ecache_flushaddr"); 1811 } 1812 } 1813 1814 /* 1815 * Virtual available next. 1816 */ 1817 ASSERT(virt_avail != NULL); 1818 memlist_free_list(virt_avail); 1819 virt_avail = memlist; 1820 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1821 1822 /* 1823 * Last chance to ask our booter questions .. 1824 */ 1825 } 1826 1827 1828 /* 1829 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1830 * allocations have been performed. We can't call it in startup_bop_gone 1831 * since later operations can cause obp to allocate more memory. 1832 */ 1833 void 1834 startup_fixup_physavail(void) 1835 { 1836 struct memlist *cur; 1837 1838 /* 1839 * take the most current snapshot we can by calling mem-update 1840 */ 1841 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1842 &boot_physavail, &boot_physavail_len, 1843 &boot_virtavail, &boot_virtavail_len); 1844 1845 /* 1846 * Copy phys_avail list, again. 1847 * Both the kernel/boot and the prom have been allocating 1848 * from the original list we copied earlier. 1849 */ 1850 cur = memlist; 1851 (void) copy_physavail(boot_physavail, boot_physavail_len, 1852 &memlist, 0, 0); 1853 1854 /* 1855 * Add any extra memory after e_text we added to the phys_avail list 1856 * back to the old list. 1857 */ 1858 if (extra_etpg) 1859 memlist_add(va_to_pa(extra_etva), mmu_ptob(extra_etpg), 1860 &memlist, &cur); 1861 if (ndata_remain_sz >= MMU_PAGESIZE) 1862 memlist_add(va_to_pa(nalloc_base), 1863 (uint64_t)ndata_remain_sz, &memlist, &cur); 1864 1865 /* 1866 * There isn't any bounds checking on the memlist area 1867 * so ensure it hasn't overgrown. 1868 */ 1869 if ((caddr_t)memlist > (caddr_t)memlist_end) 1870 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1871 1872 /* 1873 * The kernel removes the pages that were allocated for it from 1874 * the freelist, but we now have to find any -extra- pages that 1875 * the prom has allocated for it's own book-keeping, and remove 1876 * them from the freelist too. sigh. 1877 */ 1878 fix_prom_pages(phys_avail, cur); 1879 1880 ASSERT(phys_avail != NULL); 1881 memlist_free_list(phys_avail); 1882 phys_avail = cur; 1883 1884 /* 1885 * We're done with boot. Just after this point in time, boot 1886 * gets unmapped, so we can no longer rely on its services. 1887 * Zero the bootops to indicate this fact. 1888 */ 1889 bootops = (struct bootops *)NULL; 1890 BOOTOPS_GONE(); 1891 } 1892 1893 static void 1894 startup_vm(void) 1895 { 1896 size_t i; 1897 struct segmap_crargs a; 1898 struct segkpm_crargs b; 1899 1900 uint64_t avmem; 1901 caddr_t va; 1902 pgcnt_t max_phys_segkp; 1903 int mnode; 1904 1905 extern int use_brk_lpg, use_stk_lpg; 1906 1907 /* 1908 * get prom's mappings, create hments for them and switch 1909 * to the kernel context. 1910 */ 1911 hat_kern_setup(); 1912 1913 /* 1914 * Take over trap table 1915 */ 1916 setup_trap_table(); 1917 1918 /* 1919 * Install the va>tte handler, so that the prom can handle 1920 * misses and understand the kernel table layout in case 1921 * we need call into the prom. 1922 */ 1923 install_va_to_tte(); 1924 1925 /* 1926 * Set a flag to indicate that the tba has been taken over. 1927 */ 1928 tba_taken_over = 1; 1929 1930 /* initialize MMU primary context register */ 1931 mmu_init_kcontext(); 1932 1933 /* 1934 * The boot cpu can now take interrupts, x-calls, x-traps 1935 */ 1936 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1937 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1938 1939 /* 1940 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1941 */ 1942 tbr_wr_addr_inited = 1; 1943 1944 /* 1945 * Initialize VM system, and map kernel address space. 1946 */ 1947 kvm_init(); 1948 1949 /* 1950 * XXX4U: previously, we initialized and turned on 1951 * the caches at this point. But of course we have 1952 * nothing to do, as the prom has already done this 1953 * for us -- main memory must be E$able at all times. 1954 */ 1955 1956 /* 1957 * If the following is true, someone has patched 1958 * phsymem to be less than the number of pages that 1959 * the system actually has. Remove pages until system 1960 * memory is limited to the requested amount. Since we 1961 * have allocated page structures for all pages, we 1962 * correct the amount of memory we want to remove 1963 * by the size of the memory used to hold page structures 1964 * for the non-used pages. 1965 */ 1966 if (physmem < npages) { 1967 pgcnt_t diff, off; 1968 struct page *pp; 1969 struct seg kseg; 1970 1971 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1972 1973 off = 0; 1974 diff = npages - physmem; 1975 diff -= mmu_btopr(diff * sizeof (struct page)); 1976 kseg.s_as = &kas; 1977 while (diff--) { 1978 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1979 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1980 &kseg, (caddr_t)off); 1981 if (pp == NULL) 1982 cmn_err(CE_PANIC, "limited physmem too much!"); 1983 page_io_unlock(pp); 1984 page_downgrade(pp); 1985 availrmem--; 1986 off += MMU_PAGESIZE; 1987 } 1988 } 1989 1990 /* 1991 * When printing memory, show the total as physmem less 1992 * that stolen by a debugger. 1993 */ 1994 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1995 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1996 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1997 1998 avmem = (uint64_t)freemem << PAGESHIFT; 1999 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 2000 2001 /* 2002 * For small memory systems disable automatic large pages. 2003 */ 2004 if (physmem < privm_lpg_min_physmem) { 2005 use_brk_lpg = 0; 2006 use_stk_lpg = 0; 2007 } 2008 2009 /* 2010 * Perform platform specific freelist processing 2011 */ 2012 if (&plat_freelist_process) { 2013 for (mnode = 0; mnode < max_mem_nodes; mnode++) 2014 if (mem_node_config[mnode].exists) 2015 plat_freelist_process(mnode); 2016 } 2017 2018 /* 2019 * Initialize the segkp segment type. We position it 2020 * after the configured tables and buffers (whose end 2021 * is given by econtig) and before V_WKBASE_ADDR. 2022 * Also in this area is segkmap (size SEGMAPSIZE). 2023 */ 2024 2025 /* XXX - cache alignment? */ 2026 va = (caddr_t)SEGKPBASE; 2027 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 2028 2029 max_phys_segkp = (physmem * 2); 2030 2031 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 2032 segkpsize = btop(SEGKPDEFSIZE); 2033 cmn_err(CE_WARN, "Illegal value for segkpsize. " 2034 "segkpsize has been reset to %ld pages", segkpsize); 2035 } 2036 2037 i = ptob(MIN(segkpsize, max_phys_segkp)); 2038 2039 rw_enter(&kas.a_lock, RW_WRITER); 2040 if (seg_attach(&kas, va, i, segkp) < 0) 2041 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 2042 if (segkp_create(segkp) != 0) 2043 cmn_err(CE_PANIC, "startup: segkp_create failed"); 2044 rw_exit(&kas.a_lock); 2045 2046 /* 2047 * kpm segment 2048 */ 2049 segmap_kpm = kpm_enable && 2050 segmap_kpm && PAGESIZE == MAXBSIZE; 2051 2052 if (kpm_enable) { 2053 rw_enter(&kas.a_lock, RW_WRITER); 2054 2055 /* 2056 * The segkpm virtual range range is larger than the 2057 * actual physical memory size and also covers gaps in 2058 * the physical address range for the following reasons: 2059 * . keep conversion between segkpm and physical addresses 2060 * simple, cheap and unambiguous. 2061 * . avoid extension/shrink of the the segkpm in case of DR. 2062 * . avoid complexity for handling of virtual addressed 2063 * caches, segkpm and the regular mapping scheme must be 2064 * kept in sync wrt. the virtual color of mapped pages. 2065 * Any accesses to virtual segkpm ranges not backed by 2066 * physical memory will fall through the memseg pfn hash 2067 * and will be handled in segkpm_fault. 2068 * Additional kpm_size spaces needed for vac alias prevention. 2069 */ 2070 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 2071 segkpm) < 0) 2072 cmn_err(CE_PANIC, "cannot attach segkpm"); 2073 2074 b.prot = PROT_READ | PROT_WRITE; 2075 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 2076 2077 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 2078 panic("segkpm_create segkpm"); 2079 2080 rw_exit(&kas.a_lock); 2081 2082 mach_kpm_init(); 2083 } 2084 2085 if (!segzio_fromheap) { 2086 size_t size; 2087 size_t physmem_b = mmu_ptob(physmem); 2088 2089 /* size is in bytes, segziosize is in pages */ 2090 if (segziosize == 0) { 2091 size = physmem_b; 2092 } else { 2093 size = mmu_ptob(segziosize); 2094 } 2095 2096 if (size < SEGZIOMINSIZE) { 2097 size = SEGZIOMINSIZE; 2098 } else if (size > SEGZIOMAXSIZE) { 2099 size = SEGZIOMAXSIZE; 2100 /* 2101 * On 64-bit x86, we only have 2TB of KVA. This exists 2102 * for parity with x86. 2103 * 2104 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2105 * doesn't consume all of KVA. However, if we have a 2106 * system that has more thant 512gb of physical memory, 2107 * we can actually consume about half of the difference 2108 * between 512gb and the rest of the available physical 2109 * memory. 2110 */ 2111 if (physmem_b > SEGZIOMAXSIZE) { 2112 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2113 } 2114 } 2115 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2116 /* put the base of the ZIO segment after the kpm segment */ 2117 segzio_base = kpm_vbase + (kpm_size * vac_colors); 2118 PRM_DEBUG(segziosize); 2119 PRM_DEBUG(segzio_base); 2120 2121 /* 2122 * On some platforms, kvm_init is called after the kpm 2123 * sizes have been determined. On SPARC, kvm_init is called 2124 * before, so we have to attach the kzioseg after kvm is 2125 * initialized, otherwise we'll try to allocate from the boot 2126 * area since the kernel heap hasn't yet been configured. 2127 */ 2128 rw_enter(&kas.a_lock, RW_WRITER); 2129 2130 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2131 &kzioseg); 2132 (void) segkmem_zio_create(&kzioseg); 2133 2134 /* create zio area covering new segment */ 2135 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2136 2137 rw_exit(&kas.a_lock); 2138 } 2139 2140 2141 /* 2142 * Now create generic mapping segment. This mapping 2143 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2144 * virtual address is greater than the amount of free 2145 * memory that is available, then we trim back the 2146 * segment size to that amount 2147 */ 2148 va = (caddr_t)SEGMAPBASE; 2149 2150 /* 2151 * 1201049: segkmap base address must be MAXBSIZE aligned 2152 */ 2153 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2154 2155 /* 2156 * Set size of segmap to percentage of freemem at boot, 2157 * but stay within the allowable range 2158 * Note we take percentage before converting from pages 2159 * to bytes to avoid an overflow on 32-bit kernels. 2160 */ 2161 i = mmu_ptob((freemem * segmap_percent) / 100); 2162 2163 if (i < MINMAPSIZE) 2164 i = MINMAPSIZE; 2165 2166 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2167 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2168 2169 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2170 2171 rw_enter(&kas.a_lock, RW_WRITER); 2172 if (seg_attach(&kas, va, i, segkmap) < 0) 2173 cmn_err(CE_PANIC, "cannot attach segkmap"); 2174 2175 a.prot = PROT_READ | PROT_WRITE; 2176 a.shmsize = shm_alignment; 2177 a.nfreelist = 0; /* use segmap driver defaults */ 2178 2179 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2180 panic("segmap_create segkmap"); 2181 rw_exit(&kas.a_lock); 2182 2183 segdev_init(); 2184 } 2185 2186 static void 2187 startup_end(void) 2188 { 2189 if ((caddr_t)memlist > (caddr_t)memlist_end) 2190 panic("memlist overflow 2"); 2191 memlist_free_block((caddr_t)memlist, 2192 ((caddr_t)memlist_end - (caddr_t)memlist)); 2193 memlist = NULL; 2194 2195 /* enable page_relocation since OBP is now done */ 2196 page_relocate_ready = 1; 2197 2198 /* 2199 * Perform tasks that get done after most of the VM 2200 * initialization has been done but before the clock 2201 * and other devices get started. 2202 */ 2203 kern_setup1(); 2204 2205 /* 2206 * Intialize the VM arenas for allocating physically 2207 * contiguus memory chunk for interrupt queues snd 2208 * allocate/register boot cpu's queues, if any and 2209 * allocate dump buffer for sun4v systems to store 2210 * extra crash information during crash dump 2211 */ 2212 contig_mem_init(); 2213 mach_descrip_init(); 2214 cpu_intrq_setup(CPU); 2215 cpu_intrq_register(CPU); 2216 mach_htraptrace_setup(CPU->cpu_id); 2217 mach_htraptrace_configure(CPU->cpu_id); 2218 mach_dump_buffer_init(); 2219 2220 /* 2221 * Initialize interrupt related stuff 2222 */ 2223 cpu_intr_alloc(CPU, NINTR_THREADS); 2224 2225 (void) splzs(); /* allow hi clock ints but not zs */ 2226 2227 /* 2228 * Initialize errors. 2229 */ 2230 error_init(); 2231 2232 /* 2233 * Note that we may have already used kernel bcopy before this 2234 * point - but if you really care about this, adb the use_hw_* 2235 * variables to 0 before rebooting. 2236 */ 2237 mach_hw_copy_limit(); 2238 2239 /* 2240 * Install the "real" preemption guards before DDI services 2241 * are available. 2242 */ 2243 (void) prom_set_preprom(kern_preprom); 2244 (void) prom_set_postprom(kern_postprom); 2245 CPU->cpu_m.mutex_ready = 1; 2246 2247 /* 2248 * Initialize segnf (kernel support for non-faulting loads). 2249 */ 2250 segnf_init(); 2251 2252 /* 2253 * Configure the root devinfo node. 2254 */ 2255 configure(); /* set up devices */ 2256 mach_cpu_halt_idle(); 2257 } 2258 2259 2260 void 2261 post_startup(void) 2262 { 2263 #ifdef PTL1_PANIC_DEBUG 2264 extern void init_ptl1_thread(void); 2265 #endif /* PTL1_PANIC_DEBUG */ 2266 extern void abort_sequence_init(void); 2267 2268 /* 2269 * Set the system wide, processor-specific flags to be passed 2270 * to userland via the aux vector for performance hints and 2271 * instruction set extensions. 2272 */ 2273 bind_hwcap(); 2274 2275 /* 2276 * Startup memory scrubber (if any) 2277 */ 2278 mach_memscrub(); 2279 2280 /* 2281 * Allocate soft interrupt to handle abort sequence. 2282 */ 2283 abort_sequence_init(); 2284 2285 /* 2286 * Configure the rest of the system. 2287 * Perform forceloading tasks for /etc/system. 2288 */ 2289 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2290 /* 2291 * ON4.0: Force /proc module in until clock interrupt handle fixed 2292 * ON4.0: This must be fixed or restated in /etc/systems. 2293 */ 2294 (void) modload("fs", "procfs"); 2295 2296 /* load machine class specific drivers */ 2297 load_mach_drivers(); 2298 2299 /* load platform specific drivers */ 2300 if (&load_platform_drivers) 2301 load_platform_drivers(); 2302 2303 /* load vis simulation module, if we are running w/fpu off */ 2304 if (!fpu_exists) { 2305 if (modload("misc", "vis") == -1) 2306 halt("Can't load vis"); 2307 } 2308 2309 mach_fpras(); 2310 2311 maxmem = freemem; 2312 2313 #ifdef PTL1_PANIC_DEBUG 2314 init_ptl1_thread(); 2315 #endif /* PTL1_PANIC_DEBUG */ 2316 2317 if (&cif_init) 2318 cif_init(); 2319 } 2320 2321 #ifdef PTL1_PANIC_DEBUG 2322 int ptl1_panic_test = 0; 2323 int ptl1_panic_xc_one_test = 0; 2324 int ptl1_panic_xc_all_test = 0; 2325 int ptl1_panic_xt_one_test = 0; 2326 int ptl1_panic_xt_all_test = 0; 2327 kthread_id_t ptl1_thread_p = NULL; 2328 kcondvar_t ptl1_cv; 2329 kmutex_t ptl1_mutex; 2330 int ptl1_recurse_count_threshold = 0x40; 2331 int ptl1_recurse_trap_threshold = 0x3d; 2332 extern void ptl1_recurse(int, int); 2333 extern void ptl1_panic_xt(int, int); 2334 2335 /* 2336 * Called once per second by timeout() to wake up 2337 * the ptl1_panic thread to see if it should cause 2338 * a trap to the ptl1_panic() code. 2339 */ 2340 /* ARGSUSED */ 2341 static void 2342 ptl1_wakeup(void *arg) 2343 { 2344 mutex_enter(&ptl1_mutex); 2345 cv_signal(&ptl1_cv); 2346 mutex_exit(&ptl1_mutex); 2347 } 2348 2349 /* 2350 * ptl1_panic cross call function: 2351 * Needed because xc_one() and xc_some() can pass 2352 * 64 bit args but ptl1_recurse() expects ints. 2353 */ 2354 static void 2355 ptl1_panic_xc(void) 2356 { 2357 ptl1_recurse(ptl1_recurse_count_threshold, 2358 ptl1_recurse_trap_threshold); 2359 } 2360 2361 /* 2362 * The ptl1 thread waits for a global flag to be set 2363 * and uses the recurse thresholds to set the stack depth 2364 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2365 * or indirectly via the cross call and cross trap functions. 2366 * 2367 * This is useful testing stack overflows and normal 2368 * ptl1_panic() states with a know stack frame. 2369 * 2370 * ptl1_recurse() is an asm function in ptl1_panic.s that 2371 * sets the {In, Local, Out, and Global} registers to a 2372 * know state on the stack and just prior to causing a 2373 * test ptl1_panic trap. 2374 */ 2375 static void 2376 ptl1_thread(void) 2377 { 2378 mutex_enter(&ptl1_mutex); 2379 while (ptl1_thread_p) { 2380 cpuset_t other_cpus; 2381 int cpu_id; 2382 int my_cpu_id; 2383 int target_cpu_id; 2384 int target_found; 2385 2386 if (ptl1_panic_test) { 2387 ptl1_recurse(ptl1_recurse_count_threshold, 2388 ptl1_recurse_trap_threshold); 2389 } 2390 2391 /* 2392 * Find potential targets for x-call and x-trap, 2393 * if any exist while preempt is disabled we 2394 * start a ptl1_panic if requested via a 2395 * globals. 2396 */ 2397 kpreempt_disable(); 2398 my_cpu_id = CPU->cpu_id; 2399 other_cpus = cpu_ready_set; 2400 CPUSET_DEL(other_cpus, CPU->cpu_id); 2401 target_found = 0; 2402 if (!CPUSET_ISNULL(other_cpus)) { 2403 /* 2404 * Pick the first one 2405 */ 2406 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2407 if (cpu_id == my_cpu_id) 2408 continue; 2409 2410 if (CPU_XCALL_READY(cpu_id)) { 2411 target_cpu_id = cpu_id; 2412 target_found = 1; 2413 break; 2414 } 2415 } 2416 ASSERT(target_found); 2417 2418 if (ptl1_panic_xc_one_test) { 2419 xc_one(target_cpu_id, 2420 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2421 } 2422 if (ptl1_panic_xc_all_test) { 2423 xc_some(other_cpus, 2424 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2425 } 2426 if (ptl1_panic_xt_one_test) { 2427 xt_one(target_cpu_id, 2428 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2429 } 2430 if (ptl1_panic_xt_all_test) { 2431 xt_some(other_cpus, 2432 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2433 } 2434 } 2435 kpreempt_enable(); 2436 (void) timeout(ptl1_wakeup, NULL, hz); 2437 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2438 } 2439 mutex_exit(&ptl1_mutex); 2440 } 2441 2442 /* 2443 * Called during early startup to create the ptl1_thread 2444 */ 2445 void 2446 init_ptl1_thread(void) 2447 { 2448 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2449 &p0, TS_RUN, 0); 2450 } 2451 #endif /* PTL1_PANIC_DEBUG */ 2452 2453 2454 /* 2455 * Add to a memory list. 2456 * start = start of new memory segment 2457 * len = length of new memory segment in bytes 2458 * memlistp = pointer to array of available memory segment structures 2459 * curmemlistp = memory list to which to add segment. 2460 */ 2461 static void 2462 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2463 struct memlist **curmemlistp) 2464 { 2465 struct memlist *new; 2466 2467 new = *memlistp; 2468 new->address = start; 2469 new->size = len; 2470 *memlistp = new + 1; 2471 2472 memlist_insert(new, curmemlistp); 2473 } 2474 2475 /* 2476 * In the case of architectures that support dynamic addition of 2477 * memory at run-time there are two cases where memsegs need to 2478 * be initialized and added to the memseg list. 2479 * 1) memsegs that are constructed at startup. 2480 * 2) memsegs that are constructed at run-time on 2481 * hot-plug capable architectures. 2482 * This code was originally part of the function kphysm_init(). 2483 */ 2484 2485 static void 2486 memseg_list_add(struct memseg *memsegp) 2487 { 2488 struct memseg **prev_memsegp; 2489 pgcnt_t num; 2490 2491 /* insert in memseg list, decreasing number of pages order */ 2492 2493 num = MSEG_NPAGES(memsegp); 2494 2495 for (prev_memsegp = &memsegs; *prev_memsegp; 2496 prev_memsegp = &((*prev_memsegp)->next)) { 2497 if (num > MSEG_NPAGES(*prev_memsegp)) 2498 break; 2499 } 2500 2501 memsegp->next = *prev_memsegp; 2502 *prev_memsegp = memsegp; 2503 2504 if (kpm_enable) { 2505 memsegp->nextpa = (memsegp->next) ? 2506 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2507 2508 if (prev_memsegp != &memsegs) { 2509 struct memseg *msp; 2510 msp = (struct memseg *)((caddr_t)prev_memsegp - 2511 offsetof(struct memseg, next)); 2512 msp->nextpa = va_to_pa(memsegp); 2513 } else { 2514 memsegspa = va_to_pa(memsegs); 2515 } 2516 } 2517 } 2518 2519 /* 2520 * PSM add_physmem_cb(). US-II and newer processors have some 2521 * flavor of the prefetch capability implemented. We exploit 2522 * this capability for optimum performance. 2523 */ 2524 #define PREFETCH_BYTES 64 2525 2526 void 2527 add_physmem_cb(page_t *pp, pfn_t pnum) 2528 { 2529 extern void prefetch_page_w(void *); 2530 2531 pp->p_pagenum = pnum; 2532 2533 /* 2534 * Prefetch one more page_t into E$. To prevent future 2535 * mishaps with the sizeof(page_t) changing on us, we 2536 * catch this on debug kernels if we can't bring in the 2537 * entire hpage with 2 PREFETCH_BYTES reads. See 2538 * also, sun4u/cpu/cpu_module.c 2539 */ 2540 /*LINTED*/ 2541 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2542 prefetch_page_w((char *)pp); 2543 } 2544 2545 /* 2546 * kphysm_init() tackles the problem of initializing physical memory. 2547 * The old startup made some assumptions about the kernel living in 2548 * physically contiguous space which is no longer valid. 2549 */ 2550 static void 2551 kphysm_init(page_t *pp, struct memseg *memsegp, pgcnt_t npages, 2552 uintptr_t kpm_pp, pgcnt_t kpm_npages) 2553 { 2554 struct memlist *pmem; 2555 struct memseg *msp; 2556 pfn_t base; 2557 pgcnt_t num; 2558 pfn_t lastseg_pages_end = 0; 2559 pgcnt_t nelem_used = 0; 2560 2561 ASSERT(page_hash != NULL && page_hashsz != 0); 2562 2563 msp = memsegp; 2564 for (pmem = phys_avail; pmem && npages; pmem = pmem->next) { 2565 2566 /* 2567 * Build the memsegs entry 2568 */ 2569 num = btop(pmem->size); 2570 if (num > npages) 2571 num = npages; 2572 npages -= num; 2573 base = btop(pmem->address); 2574 2575 msp->pages = pp; 2576 msp->epages = pp + num; 2577 msp->pages_base = base; 2578 msp->pages_end = base + num; 2579 2580 if (kpm_enable) { 2581 pfn_t pbase_a; 2582 pfn_t pend_a; 2583 pfn_t prev_pend_a; 2584 pgcnt_t nelem; 2585 2586 msp->pagespa = va_to_pa(pp); 2587 msp->epagespa = va_to_pa(pp + num); 2588 pbase_a = kpmptop(ptokpmp(base)); 2589 pend_a = kpmptop(ptokpmp(base + num - 1)) + kpmpnpgs; 2590 nelem = ptokpmp(pend_a - pbase_a); 2591 msp->kpm_nkpmpgs = nelem; 2592 msp->kpm_pbase = pbase_a; 2593 if (lastseg_pages_end) { 2594 /* 2595 * Assume phys_avail is in ascending order 2596 * of physical addresses. 2597 */ 2598 ASSERT(base + num > lastseg_pages_end); 2599 prev_pend_a = kpmptop( 2600 ptokpmp(lastseg_pages_end - 1)) + kpmpnpgs; 2601 2602 if (prev_pend_a > pbase_a) { 2603 /* 2604 * Overlap, more than one memseg may 2605 * point to the same kpm_page range. 2606 */ 2607 if (kpm_smallpages == 0) { 2608 msp->kpm_pages = 2609 (kpm_page_t *)kpm_pp - 1; 2610 kpm_pp = (uintptr_t) 2611 ((kpm_page_t *)kpm_pp 2612 + nelem - 1); 2613 } else { 2614 msp->kpm_spages = 2615 (kpm_spage_t *)kpm_pp - 1; 2616 kpm_pp = (uintptr_t) 2617 ((kpm_spage_t *)kpm_pp 2618 + nelem - 1); 2619 } 2620 nelem_used += nelem - 1; 2621 2622 } else { 2623 if (kpm_smallpages == 0) { 2624 msp->kpm_pages = 2625 (kpm_page_t *)kpm_pp; 2626 kpm_pp = (uintptr_t) 2627 ((kpm_page_t *)kpm_pp 2628 + nelem); 2629 } else { 2630 msp->kpm_spages = 2631 (kpm_spage_t *)kpm_pp; 2632 kpm_pp = (uintptr_t) 2633 ((kpm_spage_t *) 2634 kpm_pp + nelem); 2635 } 2636 nelem_used += nelem; 2637 } 2638 2639 } else { 2640 if (kpm_smallpages == 0) { 2641 msp->kpm_pages = (kpm_page_t *)kpm_pp; 2642 kpm_pp = (uintptr_t) 2643 ((kpm_page_t *)kpm_pp + nelem); 2644 } else { 2645 msp->kpm_spages = (kpm_spage_t *)kpm_pp; 2646 kpm_pp = (uintptr_t) 2647 ((kpm_spage_t *)kpm_pp + nelem); 2648 } 2649 nelem_used = nelem; 2650 } 2651 2652 if (nelem_used > kpm_npages) 2653 panic("kphysm_init: kpm_pp overflow\n"); 2654 2655 msp->kpm_pagespa = va_to_pa(msp->kpm_pages); 2656 lastseg_pages_end = msp->pages_end; 2657 } 2658 2659 memseg_list_add(msp); 2660 2661 /* 2662 * add_physmem() initializes the PSM part of the page 2663 * struct by calling the PSM back with add_physmem_cb(). 2664 * In addition it coalesces pages into larger pages as 2665 * it initializes them. 2666 */ 2667 add_physmem(pp, num, base); 2668 pp += num; 2669 msp++; 2670 } 2671 2672 build_pfn_hash(); 2673 } 2674 2675 /* 2676 * Kernel VM initialization. 2677 * Assumptions about kernel address space ordering: 2678 * (1) gap (user space) 2679 * (2) kernel text 2680 * (3) kernel data/bss 2681 * (4) gap 2682 * (5) kernel data structures 2683 * (6) gap 2684 * (7) debugger (optional) 2685 * (8) monitor 2686 * (9) gap (possibly null) 2687 * (10) dvma 2688 * (11) devices 2689 */ 2690 static void 2691 kvm_init(void) 2692 { 2693 /* 2694 * Put the kernel segments in kernel address space. 2695 */ 2696 rw_enter(&kas.a_lock, RW_WRITER); 2697 as_avlinit(&kas); 2698 2699 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2700 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2701 (void) segkmem_create(&ktextseg); 2702 2703 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2704 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2705 (void) segkmem_create(&ktexthole); 2706 2707 (void) seg_attach(&kas, (caddr_t)valloc_base, 2708 (size_t)(econtig32 - valloc_base), &kvalloc); 2709 (void) segkmem_create(&kvalloc); 2710 2711 if (kmem64_base) { 2712 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2713 (size_t)(kmem64_end - kmem64_base), &kmem64); 2714 (void) segkmem_create(&kmem64); 2715 } 2716 2717 /* 2718 * We're about to map out /boot. This is the beginning of the 2719 * system resource management transition. We can no longer 2720 * call into /boot for I/O or memory allocations. 2721 */ 2722 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2723 (void) segkmem_create(&kvseg); 2724 hblk_alloc_dynamic = 1; 2725 2726 /* 2727 * we need to preallocate pages for DR operations before enabling large 2728 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2729 */ 2730 memseg_remap_init(); 2731 2732 /* at this point we are ready to use large page heap */ 2733 segkmem_heap_lp_init(); 2734 2735 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2736 &kvseg32); 2737 (void) segkmem_create(&kvseg32); 2738 2739 /* 2740 * Create a segment for the debugger. 2741 */ 2742 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2743 (void) segkmem_create(&kdebugseg); 2744 2745 rw_exit(&kas.a_lock); 2746 } 2747 2748 char obp_tte_str[] = 2749 "h# %x constant MMU_PAGESHIFT " 2750 "h# %x constant TTE8K " 2751 "h# %x constant SFHME_SIZE " 2752 "h# %x constant SFHME_TTE " 2753 "h# %x constant HMEBLK_TAG " 2754 "h# %x constant HMEBLK_NEXT " 2755 "h# %x constant HMEBLK_MISC " 2756 "h# %x constant HMEBLK_HME1 " 2757 "h# %x constant NHMENTS " 2758 "h# %x constant HBLK_SZMASK " 2759 "h# %x constant HBLK_RANGE_SHIFT " 2760 "h# %x constant HMEBP_HBLK " 2761 "h# %x constant HMEBUCKET_SIZE " 2762 "h# %x constant HTAG_SFMMUPSZ " 2763 "h# %x constant HTAG_REHASHSZ " 2764 "h# %x constant mmu_hashcnt " 2765 "h# %p constant uhme_hash " 2766 "h# %p constant khme_hash " 2767 "h# %x constant UHMEHASH_SZ " 2768 "h# %x constant KHMEHASH_SZ " 2769 "h# %p constant KCONTEXT " 2770 "h# %p constant KHATID " 2771 "h# %x constant ASI_MEM " 2772 2773 ": PHYS-X@ ( phys -- data ) " 2774 " ASI_MEM spacex@ " 2775 "; " 2776 2777 ": PHYS-W@ ( phys -- data ) " 2778 " ASI_MEM spacew@ " 2779 "; " 2780 2781 ": PHYS-L@ ( phys -- data ) " 2782 " ASI_MEM spaceL@ " 2783 "; " 2784 2785 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2786 " 3 * MMU_PAGESHIFT + " 2787 "; " 2788 2789 ": TTE_IS_VALID ( ttep -- flag ) " 2790 " PHYS-X@ 0< " 2791 "; " 2792 2793 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2794 " dup TTE8K = if " 2795 " drop HBLK_RANGE_SHIFT " 2796 " else " 2797 " TTE_PAGE_SHIFT " 2798 " then " 2799 "; " 2800 2801 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2802 " tuck >> swap MMU_PAGESHIFT - << " 2803 "; " 2804 2805 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2806 " >> over xor swap ( hash sfmmup ) " 2807 " KHATID <> if ( hash ) " 2808 " UHMEHASH_SZ and ( bucket ) " 2809 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2810 " else ( hash ) " 2811 " KHMEHASH_SZ and ( bucket ) " 2812 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2813 " then ( hmebp ) " 2814 "; " 2815 2816 ": HME_HASH_TABLE_SEARCH " 2817 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2818 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2819 " dup if ( sfmmup hmeblkp ) ( r: hblktag ) " 2820 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2821 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2822 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2823 " else " 2824 " hmeblk_next + phys-x@ false " 2825 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2826 " then " 2827 " else " 2828 " hmeblk_next + phys-x@ false " 2829 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2830 " then " 2831 " else " 2832 " true " 2833 " then " 2834 " until r> drop " 2835 "; " 2836 2837 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2838 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2839 " HTAG_REHASHSZ << or nip ( hblktag ) " 2840 "; " 2841 2842 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2843 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2844 " TTE8K = if ( hmeblkp addr ) " 2845 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2846 " else ( hmeblkp addr ) " 2847 " drop 0 ( hmeblkp 0 ) " 2848 " then ( hmeblkp hme-index ) " 2849 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2850 " SFHME_TTE + ( ttep ) " 2851 "; " 2852 2853 ": unix-tte ( addr cnum -- false | tte-data true ) " 2854 " KCONTEXT = if ( addr ) " 2855 " KHATID ( addr khatid ) " 2856 " else ( addr ) " 2857 " drop false exit ( false ) " 2858 " then " 2859 " ( addr khatid ) " 2860 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2861 " 2dup swap i HME_HASH_SHIFT " 2862 "( addr sfmmup sfmmup addr hmeshift ) " 2863 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2864 " over i 4 pick " 2865 "( addr sfmmup hmebp sfmmup rehash addr ) " 2866 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2867 " HME_HASH_TABLE_SEARCH " 2868 "( addr sfmmup { null | hmeblkp } ) " 2869 " ?dup if ( addr sfmmup hmeblkp ) " 2870 " nip swap HBLK_TO_TTEP ( ttep ) " 2871 " dup TTE_IS_VALID if ( valid-ttep ) " 2872 " PHYS-X@ true ( tte-data true ) " 2873 " else ( invalid-tte ) " 2874 " drop false ( false ) " 2875 " then ( false | tte-data true ) " 2876 " unloop exit ( false | tte-data true ) " 2877 " then ( addr sfmmup ) " 2878 " loop ( addr sfmmup ) " 2879 " 2drop false ( false ) " 2880 "; " 2881 ; 2882 2883 void 2884 create_va_to_tte(void) 2885 { 2886 char *bp; 2887 extern int khmehash_num, uhmehash_num; 2888 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2889 2890 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2891 2892 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2893 2894 /* 2895 * Teach obp how to parse our sw ttes. 2896 */ 2897 (void) sprintf(bp, obp_tte_str, 2898 MMU_PAGESHIFT, 2899 TTE8K, 2900 sizeof (struct sf_hment), 2901 OFFSET(struct sf_hment, hme_tte), 2902 OFFSET(struct hme_blk, hblk_tag), 2903 OFFSET(struct hme_blk, hblk_nextpa), 2904 OFFSET(struct hme_blk, hblk_misc), 2905 OFFSET(struct hme_blk, hblk_hme), 2906 NHMENTS, 2907 HBLK_SZMASK, 2908 HBLK_RANGE_SHIFT, 2909 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2910 sizeof (struct hmehash_bucket), 2911 HTAG_SFMMUPSZ, 2912 HTAG_REHASHSZ, 2913 mmu_hashcnt, 2914 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2915 (caddr_t)va_to_pa((caddr_t)khme_hash), 2916 UHMEHASH_SZ, 2917 KHMEHASH_SZ, 2918 KCONTEXT, 2919 KHATID, 2920 ASI_MEM); 2921 prom_interpret(bp, 0, 0, 0, 0, 0); 2922 2923 kobj_free(bp, MMU_PAGESIZE); 2924 } 2925 2926 void 2927 install_va_to_tte(void) 2928 { 2929 /* 2930 * advise prom that he can use unix-tte 2931 */ 2932 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2933 } 2934 2935 2936 /* 2937 * Because kmdb links prom_stdout_is_framebuffer into its own 2938 * module, we add "device-type=display" here for /os-io node, so that 2939 * prom_stdout_is_framebuffer still works corrrectly after /os-io node 2940 * is registered into OBP. 2941 */ 2942 static char *create_node = 2943 "\" /\" find-device " 2944 "new-device " 2945 "\" os-io\" device-name " 2946 "\" display\" device-type " 2947 ": cb-r/w ( adr,len method$ -- #read/#written ) " 2948 " 2>r swap 2 2r> ['] $callback catch if " 2949 " 2drop 3drop 0 " 2950 " then " 2951 "; " 2952 ": read ( adr,len -- #read ) " 2953 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 2954 " ( retN ... ret1 N ) " 2955 " ?dup if " 2956 " swap >r 1- 0 ?do drop loop r> " 2957 " else " 2958 " -2 " 2959 " then " 2960 "; " 2961 ": write ( adr,len -- #written ) " 2962 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 2963 " ( retN ... ret1 N ) " 2964 " ?dup if " 2965 " swap >r 1- 0 ?do drop loop r> " 2966 " else " 2967 " 0 " 2968 " then " 2969 "; " 2970 ": poll-tty ( -- ) ; " 2971 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 2972 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 2973 ": cb-give/take ( $method -- ) " 2974 " 0 -rot ['] $callback catch ?dup if " 2975 " >r 2drop 2drop r> throw " 2976 " else " 2977 " 0 ?do drop loop " 2978 " then " 2979 "; " 2980 ": give ( -- ) \" exit-input\" cb-give/take ; " 2981 ": take ( -- ) \" enter-input\" cb-give/take ; " 2982 ": open ( -- ok? ) true ; " 2983 ": close ( -- ) ; " 2984 "finish-device " 2985 "device-end "; 2986 2987 /* 2988 * Create the OBP input/output node (FCode serial driver). 2989 * It is needed for both USB console keyboard and for 2990 * the kernel terminal emulator. It is too early to check for a 2991 * kernel console compatible framebuffer now, so we create this 2992 * so that we're ready if we need to enable kernel terminal emulation. 2993 * 2994 * When the USB software takes over the input device at the time 2995 * consconfig runs, OBP's stdin is redirected to this node. 2996 * Whenever the FORTH user interface is used after this switch, 2997 * the node will call back into the kernel for console input. 2998 * If a serial device such as ttya or a UART with a Type 5 keyboard 2999 * attached is used, OBP takes over the serial device when the system 3000 * goes to the debugger after the system is booted. This sharing 3001 * of the relatively simple serial device is difficult but possible. 3002 * Sharing the USB host controller is impossible due its complexity. 3003 * 3004 * Similarly to USB keyboard input redirection, after consconfig_dacf 3005 * configures a kernel console framebuffer as the standard output 3006 * device, OBP's stdout is switched to to vector through the 3007 * /os-io node into the kernel terminal emulator. 3008 */ 3009 static void 3010 startup_create_io_node(void) 3011 { 3012 prom_interpret(create_node, 0, 0, 0, 0, 0); 3013 } 3014 3015 3016 static void 3017 do_prom_version_check(void) 3018 { 3019 int i; 3020 pnode_t node; 3021 char buf[64]; 3022 static char drev[] = "Down-rev firmware detected%s\n" 3023 "\tPlease upgrade to the following minimum version:\n" 3024 "\t\t%s\n"; 3025 3026 i = prom_version_check(buf, sizeof (buf), &node); 3027 3028 if (i == PROM_VER64_OK) 3029 return; 3030 3031 if (i == PROM_VER64_UPGRADE) { 3032 cmn_err(CE_WARN, drev, "", buf); 3033 3034 #ifdef DEBUG 3035 prom_enter_mon(); /* Type 'go' to continue */ 3036 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3037 return; 3038 #else 3039 halt(0); 3040 #endif 3041 } 3042 3043 /* 3044 * The other possibility is that this is a server running 3045 * good firmware, but down-rev firmware was detected on at 3046 * least one other cpu board. We just complain if we see 3047 * that. 3048 */ 3049 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3050 } 3051 3052 static void 3053 kpm_init() 3054 { 3055 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 3056 kpm_pgsz = 1ull << kpm_pgshft; 3057 kpm_pgoff = kpm_pgsz - 1; 3058 kpmp2pshft = kpm_pgshft - PAGESHIFT; 3059 kpmpnpgs = 1 << kpmp2pshft; 3060 ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); 3061 } 3062 3063 void 3064 kpm_npages_setup(int memblocks) 3065 { 3066 /* 3067 * npages can be scattered in a maximum of 'memblocks' 3068 */ 3069 kpm_npages = ptokpmpr(npages) + memblocks; 3070 } 3071 3072 /* 3073 * Must be defined in platform dependent code. 3074 */ 3075 extern caddr_t modtext; 3076 extern size_t modtext_sz; 3077 extern caddr_t moddata; 3078 3079 #define HEAPTEXT_ARENA(addr) \ 3080 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3081 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3082 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3083 3084 #define HEAPTEXT_OVERSIZED(addr) \ 3085 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3086 3087 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3088 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3089 kmutex_t texthole_lock; 3090 3091 char kern_bootargs[OBP_MAXPATHLEN]; 3092 3093 void 3094 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3095 { 3096 uintptr_t addr, limit; 3097 3098 addr = HEAPTEXT_BASE; 3099 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3100 3101 /* 3102 * Before we initialize the text_arena, we want to punch holes in the 3103 * underlying heaptext_arena. This guarantees that for any text 3104 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3105 */ 3106 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3107 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3108 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3109 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3110 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3111 } 3112 3113 /* 3114 * Allocate one page at the oversize to break up the text region 3115 * from the oversized region. 3116 */ 3117 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3118 (void *)limit, (void *)(limit + PAGESIZE), 3119 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3120 3121 *text_arena = vmem_create("module_text", modtext, modtext_sz, 3122 sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3123 heaptext_arena, 0, VM_SLEEP); 3124 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3125 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3126 } 3127 3128 caddr_t 3129 kobj_text_alloc(vmem_t *arena, size_t size) 3130 { 3131 caddr_t rval, better; 3132 3133 /* 3134 * First, try a sleeping allocation. 3135 */ 3136 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3137 3138 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3139 return (rval); 3140 3141 /* 3142 * We didn't get the area that we wanted. We're going to try to do an 3143 * allocation with explicit constraints. 3144 */ 3145 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3146 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3147 VM_NOSLEEP | VM_BESTFIT); 3148 3149 if (better != NULL) { 3150 /* 3151 * That worked. Free our first attempt and return. 3152 */ 3153 vmem_free(arena, rval, size); 3154 return (better); 3155 } 3156 3157 /* 3158 * That didn't work; we'll have to return our first attempt. 3159 */ 3160 return (rval); 3161 } 3162 3163 caddr_t 3164 kobj_texthole_alloc(caddr_t addr, size_t size) 3165 { 3166 int arena = HEAPTEXT_ARENA(addr); 3167 char c[30]; 3168 uintptr_t base; 3169 3170 if (HEAPTEXT_OVERSIZED(addr)) { 3171 /* 3172 * If this is an oversized allocation, there is no text hole 3173 * available for it; return NULL. 3174 */ 3175 return (NULL); 3176 } 3177 3178 mutex_enter(&texthole_lock); 3179 3180 if (texthole_arena[arena] == NULL) { 3181 ASSERT(texthole_source[arena] == NULL); 3182 3183 if (arena == 0) { 3184 texthole_source[0] = vmem_create("module_text_holesrc", 3185 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3186 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3187 0, VM_SLEEP); 3188 } else { 3189 base = HEAPTEXT_BASE + 3190 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3191 3192 (void) snprintf(c, sizeof (c), 3193 "heaptext_holesrc_%d", arena); 3194 3195 texthole_source[arena] = vmem_create(c, (void *)base, 3196 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3197 0, VM_SLEEP); 3198 } 3199 3200 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3201 3202 texthole_arena[arena] = vmem_create(c, NULL, 0, 3203 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3204 texthole_source[arena], 0, VM_SLEEP); 3205 } 3206 3207 mutex_exit(&texthole_lock); 3208 3209 ASSERT(texthole_arena[arena] != NULL); 3210 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3211 return (vmem_alloc(texthole_arena[arena], size, 3212 VM_BESTFIT | VM_NOSLEEP)); 3213 } 3214 3215 void 3216 kobj_texthole_free(caddr_t addr, size_t size) 3217 { 3218 int arena = HEAPTEXT_ARENA(addr); 3219 3220 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3221 ASSERT(texthole_arena[arena] != NULL); 3222 vmem_free(texthole_arena[arena], addr, size); 3223 } 3224