1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 * Copyright 2018 Joyent, Inc. 26 * Copyright 2019 Peter Tribble. 27 */ 28 29 #include <sys/machsystm.h> 30 #include <sys/archsystm.h> 31 #include <sys/vm.h> 32 #include <sys/cpu.h> 33 #include <sys/atomic.h> 34 #include <sys/reboot.h> 35 #include <sys/kdi.h> 36 #include <sys/bootconf.h> 37 #include <sys/memlist_plat.h> 38 #include <sys/memlist_impl.h> 39 #include <sys/prom_plat.h> 40 #include <sys/prom_isa.h> 41 #include <sys/autoconf.h> 42 #include <sys/ivintr.h> 43 #include <sys/fpu/fpusystm.h> 44 #include <sys/iommutsb.h> 45 #include <vm/vm_dep.h> 46 #include <vm/seg_dev.h> 47 #include <vm/seg_kmem.h> 48 #include <vm/seg_kpm.h> 49 #include <vm/seg_map.h> 50 #include <vm/seg_kp.h> 51 #include <sys/sysconf.h> 52 #include <vm/hat_sfmmu.h> 53 #include <sys/kobj.h> 54 #include <sys/sun4asi.h> 55 #include <sys/clconf.h> 56 #include <sys/platform_module.h> 57 #include <sys/panic.h> 58 #include <sys/cpu_sgnblk_defs.h> 59 #include <sys/clock.h> 60 #include <sys/cmn_err.h> 61 #include <sys/dumphdr.h> 62 #include <sys/promif.h> 63 #include <sys/prom_debug.h> 64 #include <sys/traptrace.h> 65 #include <sys/memnode.h> 66 #include <sys/mem_cage.h> 67 #include <sys/mmu.h> 68 #include <sys/swap.h> 69 70 extern void setup_trap_table(void); 71 extern int cpu_intrq_setup(struct cpu *); 72 extern void cpu_intrq_register(struct cpu *); 73 extern void contig_mem_init(void); 74 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 75 extern void mach_dump_buffer_init(void); 76 extern void mach_descrip_init(void); 77 extern void mach_descrip_startup_fini(void); 78 extern void mach_memscrub(void); 79 extern void mach_fpras(void); 80 extern void mach_cpu_halt_idle(void); 81 extern void mach_hw_copy_limit(void); 82 extern void load_mach_drivers(void); 83 extern void load_tod_module(void); 84 #pragma weak load_tod_module 85 86 extern int ndata_alloc_mmfsa(struct memlist *ndata); 87 #pragma weak ndata_alloc_mmfsa 88 89 extern void cif_init(void); 90 #pragma weak cif_init 91 92 extern void parse_idprom(void); 93 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 94 extern void mem_config_init(void); 95 extern void memseg_remap_init(void); 96 97 extern void mach_kpm_init(void); 98 extern void pcf_init(); 99 extern int size_pse_array(pgcnt_t, int); 100 extern void pg_init(); 101 102 /* 103 * External Data: 104 */ 105 extern int vac_size; /* cache size in bytes */ 106 extern uint_t vac_mask; /* VAC alignment consistency mask */ 107 extern uint_t vac_colors; 108 109 /* 110 * Global Data Definitions: 111 */ 112 113 /* 114 * XXX - Don't port this to new architectures 115 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 116 * 'romp' has no use with a prom with an IEEE 1275 client interface. 117 * The driver doesn't use the value, but it depends on the symbol. 118 */ 119 void *romp; /* veritas driver won't load without romp 4154976 */ 120 /* 121 * Declare these as initialized data so we can patch them. 122 */ 123 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 124 pgcnt_t segkpsize = 125 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 126 uint_t segmap_percent = 6; /* Size of segmap segment */ 127 128 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 129 int vac_copyback = 1; 130 char *cache_mode = NULL; 131 int use_mix = 1; 132 int prom_debug = 0; 133 134 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 135 uint_t tba_taken_over = 0; 136 137 caddr_t s_text; /* start of kernel text segment */ 138 caddr_t e_text; /* end of kernel text segment */ 139 caddr_t s_data; /* start of kernel data segment */ 140 caddr_t e_data; /* end of kernel data segment */ 141 142 caddr_t modtext; /* beginning of module text */ 143 size_t modtext_sz; /* size of module text */ 144 caddr_t moddata; /* beginning of module data reserve */ 145 caddr_t e_moddata; /* end of module data reserve */ 146 147 /* 148 * End of first block of contiguous kernel in 32-bit virtual address space 149 */ 150 caddr_t econtig32; /* end of first blk of contiguous kernel */ 151 152 caddr_t ncbase; /* beginning of non-cached segment */ 153 caddr_t ncend; /* end of non-cached segment */ 154 155 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 156 caddr_t nalloc_base; /* beginning of nucleus allocation */ 157 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 158 caddr_t valloc_base; /* beginning of kvalloc segment */ 159 160 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 161 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 162 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 163 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 164 int kmem64_szc; /* page size code */ 165 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 166 167 uintptr_t shm_alignment; /* VAC address consistency modulus */ 168 struct memlist *phys_install; /* Total installed physical memory */ 169 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 170 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 171 struct memlist *nopp_list; /* pages with no backing page structs */ 172 struct memlist ndata; /* memlist of nucleus allocatable memory */ 173 int memexp_flag; /* memory expansion card flag */ 174 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 175 pgcnt_t obp_pages; /* Physical pages used by OBP */ 176 177 /* 178 * VM data structures 179 */ 180 long page_hashsz; /* Size of page hash table (power of two) */ 181 unsigned int page_hashsz_shift; /* log2(page_hashsz) */ 182 struct page *pp_base; /* Base of system page struct array */ 183 size_t pp_sz; /* Size in bytes of page struct array */ 184 struct page **page_hash; /* Page hash table */ 185 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 186 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 187 int pse_shift; /* log2(pse_table_size) */ 188 struct seg ktextseg; /* Segment used for kernel executable image */ 189 struct seg kvalloc; /* Segment used for "valloc" mapping */ 190 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 191 struct seg ktexthole; /* Segment used for nucleus text hole */ 192 struct seg kmapseg; /* Segment used for generic kernel mappings */ 193 struct seg kpmseg; /* Segment used for physical mapping */ 194 struct seg kdebugseg; /* Segment used for the kernel debugger */ 195 196 void *kpm_pp_base; /* Base of system kpm_page array */ 197 size_t kpm_pp_sz; /* Size of system kpm_page array */ 198 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 199 200 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 201 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 202 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 203 204 int segzio_fromheap = 0; /* zio allocations occur from heap */ 205 caddr_t segzio_base; /* Base address of segzio */ 206 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 207 208 /* 209 * A static DR page_t VA map is reserved that can map the page structures 210 * for a domain's entire RA space. The pages that backs this space are 211 * dynamically allocated and need not be physically contiguous. The DR 212 * map size is derived from KPM size. 213 */ 214 int ppvm_enable = 0; /* Static virtual map for page structs */ 215 page_t *ppvm_base; /* Base of page struct map */ 216 pgcnt_t ppvm_size = 0; /* Size of page struct map */ 217 218 /* 219 * debugger pages (if allocated) 220 */ 221 struct vnode kdebugvp; 222 223 /* 224 * VA range available to the debugger 225 */ 226 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 227 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 228 229 /* 230 * Segment for relocated kernel structures in 64-bit large RAM kernels 231 */ 232 struct seg kmem64; 233 234 struct memseg *memseg_free; 235 236 struct vnode unused_pages_vp; 237 238 /* 239 * VM data structures allocated early during boot. 240 */ 241 size_t pagehash_sz; 242 uint64_t memlist_sz; 243 244 char tbr_wr_addr_inited = 0; 245 246 caddr_t mpo_heap32_buf = NULL; 247 size_t mpo_heap32_bufsz = 0; 248 249 /* 250 * Static Routines: 251 */ 252 static int ndata_alloc_memseg(struct memlist *, size_t); 253 static void memlist_new(uint64_t, uint64_t, struct memlist **); 254 static void memlist_add(uint64_t, uint64_t, 255 struct memlist **, struct memlist **); 256 static void kphysm_init(void); 257 static void kvm_init(void); 258 static void install_kmem64_tte(void); 259 260 static void startup_init(void); 261 static void startup_memlist(void); 262 static void startup_modules(void); 263 static void startup_bop_gone(void); 264 static void startup_vm(void); 265 static void startup_end(void); 266 static void setup_cage_params(void); 267 static void startup_create_io_node(void); 268 269 static pgcnt_t npages; 270 static struct memlist *memlist; 271 void *memlist_end; 272 273 static pgcnt_t bop_alloc_pages; 274 static caddr_t hblk_base; 275 uint_t hblk_alloc_dynamic = 0; 276 uint_t hblk1_min = H1MIN; 277 278 279 /* 280 * After receiving a thermal interrupt, this is the number of seconds 281 * to delay before shutting off the system, assuming 282 * shutdown fails. Use /etc/system to change the delay if this isn't 283 * large enough. 284 */ 285 int thermal_powerdown_delay = 1200; 286 287 /* 288 * Used to hold off page relocations into the cage until OBP has completed 289 * its boot-time handoff of its resources to the kernel. 290 */ 291 int page_relocate_ready = 0; 292 293 /* 294 * Indicate if kmem64 allocation was done in small chunks 295 */ 296 int kmem64_smchunks = 0; 297 298 /* 299 * Enable some debugging messages concerning memory usage... 300 */ 301 #ifdef DEBUGGING_MEM 302 static int debugging_mem; 303 static void 304 printmemlist(char *title, struct memlist *list) 305 { 306 if (!debugging_mem) 307 return; 308 309 printf("%s\n", title); 310 311 while (list) { 312 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 313 (uint32_t)(list->ml_address >> 32), 314 (uint32_t)list->ml_address, 315 (uint32_t)(list->ml_size >> 32), 316 (uint32_t)(list->ml_size)); 317 list = list->ml_next; 318 } 319 } 320 321 void 322 printmemseg(struct memseg *memseg) 323 { 324 if (!debugging_mem) 325 return; 326 327 printf("memseg\n"); 328 329 while (memseg) { 330 prom_printf("\tpage = 0x%p, epage = 0x%p, " 331 "pfn = 0x%x, epfn = 0x%x\n", 332 memseg->pages, memseg->epages, 333 memseg->pages_base, memseg->pages_end); 334 memseg = memseg->next; 335 } 336 } 337 338 #define debug_pause(str) halt((str)) 339 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 340 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 341 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 342 #define MPRINTF3(str, a, b, c) \ 343 if (debugging_mem) prom_printf((str), (a), (b), (c)) 344 #else /* DEBUGGING_MEM */ 345 #define MPRINTF(str) 346 #define MPRINTF1(str, a) 347 #define MPRINTF2(str, a, b) 348 #define MPRINTF3(str, a, b, c) 349 #endif /* DEBUGGING_MEM */ 350 351 352 /* 353 * 354 * Kernel's Virtual Memory Layout. 355 * /-----------------------\ 356 * 0xFFFFFFFF.FFFFFFFF -| |- 357 * | OBP's virtual page | 358 * | tables | 359 * 0xFFFFFFFC.00000000 -|-----------------------|- 360 * : : 361 * : : 362 * -|-----------------------|- 363 * | segzio | (base and size vary) 364 * 0xFFFFFE00.00000000 -|-----------------------|- 365 * | | Ultrasparc I/II support 366 * | segkpm segment | up to 2TB of physical 367 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 368 * | | 369 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 370 * : : 371 * : : 372 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 373 * | | ^ 374 * | UltraSPARC I/II call | | 375 * | bug requires an extra | | 376 * | 4 GB of space between | | 377 * | hole and used RAM | | 378 * | | | 379 * 0xFFFFF800.00000000 -|-----------------------|- | 380 * | | | 381 * | Virtual Address Hole | UltraSPARC 382 * | on UltraSPARC I/II | I/II * ONLY * 383 * | | | 384 * 0x00000800.00000000 -|-----------------------|- | 385 * | | | 386 * | UltraSPARC I/II call | | 387 * | bug requires an extra | | 388 * | 4 GB of space between | | 389 * | hole and used RAM | | 390 * | | v 391 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 392 * : : ^ 393 * : : | 394 * |-----------------------| | 395 * | | | 396 * | ecache flush area | | 397 * | (twice largest e$) | | 398 * | | | 399 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 400 * | overmapped area | alignend_end | 401 * | (kmem64_alignsize | | 402 * | boundary) | | 403 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 404 * | | | 405 * | 64-bit kernel ONLY | | 406 * | | | 407 * | kmem64 segment | | 408 * | | | 409 * | (Relocated extra HME | Approximately 410 * | block allocations, | 1 TB of virtual 411 * | memnode freelists, | address space 412 * | HME hash buckets, | | 413 * | mml_table, kpmp_table,| | 414 * | page_t array and | | 415 * | hashblock pool to | | 416 * | avoid hard-coded | | 417 * | 32-bit vaddr | | 418 * | limitations) | | 419 * | | v 420 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 421 * | | 422 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 423 * | | 424 * 0x00000300.00000000 -|-----------------------|- SYSBASE 425 * : : 426 * : : 427 * -|-----------------------|- 428 * | | 429 * | segmap segment | SEGMAPSIZE (1/8th physmem, 430 * | | 256G MAX) 431 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 432 * : : 433 * : : 434 * -|-----------------------|- 435 * | | 436 * | segkp | SEGKPSIZE (2GB) 437 * | | 438 * | | 439 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 440 * | | 441 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 442 * | | (SEGKPBASE - 0x400000) 443 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 444 * | | (MEMSCRUBBASE - NCARGS) 445 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 446 * | | (ARGSBASE - PPMAPSIZE) 447 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 448 * | | 449 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 450 * | | 451 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 452 * : : 453 * : : 454 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 455 * | | 456 * | OBP | 457 * | | 458 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 459 * | kmdb | 460 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 461 * : : 462 * : : 463 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 464 * | | 465 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 466 * | | ~64MB) 467 * -|-----------------------| 468 * | IVSIZE | 469 * 0x00000000.70004000 -|-----------------------| 470 * | panicbuf | 471 * 0x00000000.70002000 -|-----------------------| 472 * | PAGESIZE | 473 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 474 * | boot-time | 475 * | temporary space | 476 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 477 * : : 478 * : : 479 * | | 480 * |-----------------------|- econtig32 481 * | vm structures | 482 * 0x00000000.01C00000 |-----------------------|- nalloc_end 483 * | TSBs | 484 * |-----------------------|- end/nalloc_base 485 * | kernel data & bss | 486 * 0x00000000.01800000 -|-----------------------| 487 * : nucleus text hole : 488 * 0x00000000.01400000 -|-----------------------| 489 * : : 490 * |-----------------------| 491 * | module text | 492 * |-----------------------|- e_text/modtext 493 * | kernel text | 494 * |-----------------------| 495 * | trap table (48k) | 496 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 497 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 498 * |-----------------------| 499 * | | 500 * | invalid | 501 * | | 502 * 0x00000000.00000000 _|_______________________| 503 * 504 * 505 * 506 * 32-bit User Virtual Memory Layout. 507 * /-----------------------\ 508 * | | 509 * | invalid | 510 * | | 511 * 0xFFC00000 -|-----------------------|- USERLIMIT 512 * | user stack | 513 * : : 514 * : : 515 * : : 516 * | user data | 517 * -|-----------------------|- 518 * | user text | 519 * 0x00002000 -|-----------------------|- 520 * | invalid | 521 * 0x00000000 _|_______________________| 522 * 523 * 524 * 525 * 64-bit User Virtual Memory Layout. 526 * /-----------------------\ 527 * | | 528 * | invalid | 529 * | | 530 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 531 * | user stack | 532 * : : 533 * : : 534 * : : 535 * | user data | 536 * -|-----------------------|- 537 * | user text | 538 * 0x00000000.01000000 -|-----------------------|- 539 * | invalid | 540 * 0x00000000.00000000 _|_______________________| 541 */ 542 543 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 544 extern uint64_t ecache_flush_address(void); 545 546 #pragma weak load_platform_modules 547 #pragma weak plat_startup_memlist 548 #pragma weak ecache_init_scrub_flush_area 549 #pragma weak ecache_flush_address 550 551 552 /* 553 * By default the DR Cage is enabled for maximum OS 554 * MPSS performance. Users needing to disable the cage mechanism 555 * can set this variable to zero via /etc/system. 556 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 557 * will result in loss of DR functionality. 558 * Platforms wishing to disable kernel Cage by default 559 * should do so in their set_platform_defaults() routine. 560 */ 561 int kernel_cage_enable = 1; 562 563 static void 564 setup_cage_params(void) 565 { 566 void (*func)(void); 567 568 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 569 if (func != NULL) { 570 (*func)(); 571 return; 572 } 573 574 if (kernel_cage_enable == 0) { 575 return; 576 } 577 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 578 579 if (kcage_on) { 580 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 581 } else { 582 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 583 } 584 585 } 586 587 /* 588 * Machine-dependent startup code 589 */ 590 void 591 startup(void) 592 { 593 startup_init(); 594 if (&startup_platform) 595 startup_platform(); 596 startup_memlist(); 597 startup_modules(); 598 setup_cage_params(); 599 startup_bop_gone(); 600 startup_vm(); 601 startup_end(); 602 } 603 604 struct regs sync_reg_buf; 605 uint64_t sync_tt; 606 607 void 608 sync_handler(void) 609 { 610 struct panic_trap_info ti; 611 int i; 612 613 /* 614 * Prevent trying to talk to the other CPUs since they are 615 * sitting in the prom and won't reply. 616 */ 617 for (i = 0; i < NCPU; i++) { 618 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 619 cpu[i]->cpu_flags &= ~CPU_READY; 620 cpu[i]->cpu_flags |= CPU_QUIESCED; 621 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 622 } 623 } 624 625 /* 626 * Force a serial dump, since there are no CPUs to help. 627 */ 628 dump_plat_mincpu = 0; 629 630 /* 631 * We've managed to get here without going through the 632 * normal panic code path. Try and save some useful 633 * information. 634 */ 635 if (!panicstr && (curthread->t_panic_trap == NULL)) { 636 ti.trap_type = sync_tt; 637 ti.trap_regs = &sync_reg_buf; 638 ti.trap_addr = NULL; 639 ti.trap_mmu_fsr = 0x0; 640 641 curthread->t_panic_trap = &ti; 642 } 643 644 /* 645 * If we're re-entering the panic path, update the signature 646 * block so that the SC knows we're in the second part of panic. 647 */ 648 if (panicstr) 649 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 650 651 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 652 panic("sync initiated"); 653 } 654 655 656 static void 657 startup_init(void) 658 { 659 /* 660 * We want to save the registers while we're still in OBP 661 * so that we know they haven't been fiddled with since. 662 * (In principle, OBP can't change them just because it 663 * makes a callback, but we'd rather not depend on that 664 * behavior.) 665 */ 666 char sync_str[] = 667 "warning @ warning off : sync " 668 "%%tl-c %%tstate h# %p x! " 669 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 670 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 671 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 672 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 673 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 674 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 675 "%%y h# %p l! %%tl-c %%tt h# %p x! " 676 "sync ; warning !"; 677 678 /* 679 * 20 == num of %p substrings 680 * 16 == max num of chars %p will expand to. 681 */ 682 char bp[sizeof (sync_str) + 16 * 20]; 683 684 /* 685 * Initialize ptl1 stack for the 1st CPU. 686 */ 687 ptl1_init_cpu(&cpu0); 688 689 /* 690 * Initialize the address map for cache consistent mappings 691 * to random pages; must be done after vac_size is set. 692 */ 693 ppmapinit(); 694 695 /* 696 * Initialize the PROM callback handler. 697 */ 698 init_vx_handler(); 699 700 /* 701 * have prom call sync_callback() to handle the sync and 702 * save some useful information which will be stored in the 703 * core file later. 704 */ 705 (void) sprintf((char *)bp, sync_str, 706 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 707 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 708 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 709 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 710 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 711 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 712 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 713 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 714 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 715 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 716 prom_interpret(bp, 0, 0, 0, 0, 0); 717 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 718 } 719 720 721 size_t 722 calc_pp_sz(pgcnt_t npages) 723 { 724 725 return (npages * sizeof (struct page)); 726 } 727 728 size_t 729 calc_kpmpp_sz(pgcnt_t npages) 730 { 731 732 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 733 kpm_pgsz = 1ull << kpm_pgshft; 734 kpm_pgoff = kpm_pgsz - 1; 735 kpmp2pshft = kpm_pgshft - PAGESHIFT; 736 kpmpnpgs = 1 << kpmp2pshft; 737 738 if (kpm_smallpages == 0) { 739 /* 740 * Avoid fragmentation problems in kphysm_init() 741 * by allocating for all of physical memory 742 */ 743 kpm_npages = ptokpmpr(physinstalled); 744 return (kpm_npages * sizeof (kpm_page_t)); 745 } else { 746 kpm_npages = npages; 747 return (kpm_npages * sizeof (kpm_spage_t)); 748 } 749 } 750 751 size_t 752 calc_pagehash_sz(pgcnt_t npages) 753 { 754 /* LINTED */ 755 ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page)))); 756 /* 757 * The page structure hash table size is a power of 2 758 * such that the average hash chain length is PAGE_HASHAVELEN. 759 */ 760 page_hashsz = npages / PAGE_HASHAVELEN; 761 page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1), 762 highbit(page_hashsz)); 763 page_hashsz = 1 << page_hashsz_shift; 764 return (page_hashsz * sizeof (struct page *)); 765 } 766 767 int testkmem64_smchunks = 0; 768 769 int 770 alloc_kmem64(caddr_t base, caddr_t end) 771 { 772 int i; 773 caddr_t aligned_end = NULL; 774 775 if (testkmem64_smchunks) 776 return (1); 777 778 /* 779 * Make one large memory alloc after figuring out the 64-bit size. This 780 * will enable use of the largest page size appropriate for the system 781 * architecture. 782 */ 783 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 784 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 785 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 786 size_t alloc_size, alignsize; 787 #if !defined(C_OBP) 788 unsigned long long pa; 789 #endif /* !C_OBP */ 790 791 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 792 continue; 793 alignsize = TTEBYTES(i); 794 kmem64_szc = i; 795 796 /* limit page size for small memory */ 797 if (mmu_btop(alignsize) > (npages >> 2)) 798 continue; 799 800 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 801 alloc_size = aligned_end - base; 802 #if !defined(C_OBP) 803 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 804 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 805 kmem64_pabase = pa; 806 kmem64_aligned_end = aligned_end; 807 install_kmem64_tte(); 808 break; 809 } else { 810 prom_free_phys(alloc_size, pa); 811 } 812 } 813 #else /* !C_OBP */ 814 if (prom_alloc(base, alloc_size, alignsize) == base) { 815 kmem64_pabase = va_to_pa(kmem64_base); 816 kmem64_aligned_end = aligned_end; 817 break; 818 } 819 #endif /* !C_OBP */ 820 if (i == TTE8K) { 821 #ifdef sun4v 822 /* return failure to try small allocations */ 823 return (1); 824 #else 825 prom_panic("kmem64 allocation failure"); 826 #endif 827 } 828 } 829 ASSERT(aligned_end != NULL); 830 return (0); 831 } 832 833 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 834 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 835 836 #if !defined(C_OBP) 837 /* 838 * Install a temporary tte handler in OBP for kmem64 area. 839 * 840 * We map kmem64 area with large pages before the trap table is taken 841 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 842 * the same area. Duplicate tlb entries with different page sizes 843 * cause unpredicatble behavior. To avoid this, we don't create 844 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 845 * OBP). Instead, we manage translations with a temporary va>tte-data 846 * handler (kmem64-tte). This handler is replaced by unix-tte when 847 * the trap table is taken over. 848 * 849 * The temporary handler knows the physical address of the kmem64 850 * area. It uses the prom's pgmap@ Forth word for other addresses. 851 * 852 * We have to use BOP_ALLOC() method for C-OBP platforms because 853 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 854 * sun4u platforms. On sun4u we flush tlb after trap table is taken 855 * over if we use large pages for kernel heap and kmem64. Since sun4u 856 * prom (unlike sun4v) calls va>tte-data first for client address 857 * translation prom's ttes for kmem64 can't get into TLB even if we 858 * later switch to prom's trap table again. C-OBP uses 4M pages for 859 * client mappings when possible so on all platforms we get the 860 * benefit from large mappings for kmem64 area immediately during 861 * boot. 862 * 863 * pseudo code: 864 * if (context != 0) { 865 * return false 866 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 867 * tte = tte_template + 868 * (((miss_va & pagemask) - kmem64_base)); 869 * return tte, true 870 * } else { 871 * return pgmap@ result 872 * } 873 */ 874 char kmem64_obp_str[] = 875 "h# %lx constant kmem64-base " 876 "h# %lx constant kmem64-end " 877 "h# %lx constant kmem64-pagemask " 878 "h# %lx constant kmem64-template " 879 880 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 881 " if ( addr ) " 882 " drop false exit then ( false ) " 883 " dup kmem64-base kmem64-end within if ( addr ) " 884 " kmem64-pagemask and ( addr' ) " 885 " kmem64-base - ( addr' ) " 886 " kmem64-template + ( tte ) " 887 " true ( tte true ) " 888 " else ( addr ) " 889 " pgmap@ ( tte ) " 890 " dup 0< if true else drop false then ( tte true | false ) " 891 " then ( tte true | false ) " 892 "; " 893 894 "' kmem64-tte is va>tte-data " 895 ; 896 897 static void 898 install_kmem64_tte() 899 { 900 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 901 tte_t tte; 902 903 PRM_DEBUG(kmem64_pabase); 904 PRM_DEBUG(kmem64_szc); 905 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 906 PROC_DATA | HAT_NOSYNC, kmem64_szc); 907 PRM_DEBUG(tte.ll); 908 (void) sprintf(b, kmem64_obp_str, 909 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 910 ASSERT(strlen(b) < sizeof (b)); 911 prom_interpret(b, 0, 0, 0, 0, 0); 912 } 913 #endif /* !C_OBP */ 914 915 /* 916 * As OBP takes up some RAM when the system boots, pages will already be "lost" 917 * to the system and reflected in npages by the time we see it. 918 * 919 * We only want to allocate kernel structures in the 64-bit virtual address 920 * space on systems with enough RAM to make the overhead of keeping track of 921 * an extra kernel memory segment worthwhile. 922 * 923 * Since OBP has already performed its memory allocations by this point, if we 924 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 925 * memory in the 64-bit virtual address space; otherwise keep allocations 926 * contiguous with we've mapped so far in the 32-bit virtual address space. 927 */ 928 #define MINMOVE_RAM_MB ((size_t)1900) 929 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 930 #define BYTES_TO_MB(b) ((b) / 1048576ul) 931 932 pgcnt_t tune_npages = (pgcnt_t) 933 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 934 935 #pragma weak page_set_colorequiv_arr_cpu 936 extern void page_set_colorequiv_arr_cpu(void); 937 extern void page_set_colorequiv_arr(void); 938 939 static pgcnt_t ramdisk_npages; 940 static struct memlist *old_phys_avail; 941 942 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 943 944 static void 945 startup_memlist(void) 946 { 947 size_t hmehash_sz, pagelist_sz, tt_sz; 948 size_t psetable_sz; 949 caddr_t alloc_base; 950 caddr_t memspace; 951 struct memlist *cur; 952 size_t syslimit = (size_t)SYSLIMIT; 953 size_t sysbase = (size_t)SYSBASE; 954 955 /* 956 * Initialize enough of the system to allow kmem_alloc to work by 957 * calling boot to allocate its memory until the time that 958 * kvm_init is completed. The page structs are allocated after 959 * rounding up end to the nearest page boundary; the memsegs are 960 * initialized and the space they use comes from the kernel heap. 961 * With appropriate initialization, they can be reallocated later 962 * to a size appropriate for the machine's configuration. 963 * 964 * At this point, memory is allocated for things that will never 965 * need to be freed, this used to be "valloced". This allows a 966 * savings as the pages don't need page structures to describe 967 * them because them will not be managed by the vm system. 968 */ 969 970 /* 971 * We're loaded by boot with the following configuration (as 972 * specified in the sun4u/conf/Mapfile): 973 * 974 * text: 4 MB chunk aligned on a 4MB boundary 975 * data & bss: 4 MB chunk aligned on a 4MB boundary 976 * 977 * These two chunks will eventually be mapped by 2 locked 4MB 978 * ttes and will represent the nucleus of the kernel. This gives 979 * us some free space that is already allocated, some or all of 980 * which is made available to kernel module text. 981 * 982 * The free space in the data-bss chunk is used for nucleus 983 * allocatable data structures and we reserve it using the 984 * nalloc_base and nalloc_end variables. This space is currently 985 * being used for hat data structures required for tlb miss 986 * handling operations. We align nalloc_base to a l2 cache 987 * linesize because this is the line size the hardware uses to 988 * maintain cache coherency. 989 * 512K is carved out for module data. 990 */ 991 992 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 993 e_moddata = moddata + MODDATA; 994 nalloc_base = e_moddata; 995 996 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 997 valloc_base = nalloc_base; 998 999 /* 1000 * Calculate the start of the data segment. 1001 */ 1002 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 1003 prom_panic("nucleus data overflow"); 1004 1005 PRM_DEBUG(moddata); 1006 PRM_DEBUG(nalloc_base); 1007 PRM_DEBUG(nalloc_end); 1008 1009 /* 1010 * Remember any slop after e_text so we can give it to the modules. 1011 */ 1012 PRM_DEBUG(e_text); 1013 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1014 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1015 prom_panic("nucleus text overflow"); 1016 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1017 modtext; 1018 PRM_DEBUG(modtext); 1019 PRM_DEBUG(modtext_sz); 1020 1021 init_boot_memlists(); 1022 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1023 &boot_physavail, &boot_physavail_len, 1024 &boot_virtavail, &boot_virtavail_len); 1025 1026 /* 1027 * Remember what the physically available highest page is 1028 * so that dumpsys works properly, and find out how much 1029 * memory is installed. 1030 */ 1031 installed_top_size_memlist_array(boot_physinstalled, 1032 boot_physinstalled_len, &physmax, &physinstalled); 1033 PRM_DEBUG(physinstalled); 1034 PRM_DEBUG(physmax); 1035 1036 /* Fill out memory nodes config structure */ 1037 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1038 1039 /* 1040 * npages is the maximum of available physical memory possible. 1041 * (ie. it will never be more than this) 1042 * 1043 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1044 * using phys_avail will underestimate what will end up being freed. 1045 * A better initial guess is just total memory minus the kernel text 1046 */ 1047 npages = physinstalled - btop(MMU_PAGESIZE4M); 1048 1049 /* 1050 * First allocate things that can go in the nucleus data page 1051 * (fault status, TSBs, dmv, CPUs) 1052 */ 1053 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1054 1055 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1056 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1057 1058 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1059 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1060 1061 if (ndata_alloc_dmv(&ndata) != 0) 1062 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1063 1064 if (ndata_alloc_page_mutexs(&ndata) != 0) 1065 cmn_err(CE_PANIC, 1066 "no more nucleus memory after page free lists alloc"); 1067 1068 if (ndata_alloc_hat(&ndata) != 0) 1069 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1070 1071 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1072 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1073 1074 /* 1075 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1076 * 1077 * There are comments all over the SFMMU code warning of dire 1078 * consequences if the TSBs are moved out of 32-bit space. This 1079 * is largely because the asm code uses "sethi %hi(addr)"-type 1080 * instructions which will not provide the expected result if the 1081 * address is a 64-bit one. 1082 * 1083 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1084 */ 1085 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1086 PRM_DEBUG(alloc_base); 1087 1088 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1089 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1090 PRM_DEBUG(alloc_base); 1091 1092 /* 1093 * Allocate IOMMU TSB array. We do this here so that the physical 1094 * memory gets deducted from the PROM's physical memory list. 1095 */ 1096 alloc_base = iommu_tsb_init(alloc_base); 1097 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1098 PRM_DEBUG(alloc_base); 1099 1100 /* 1101 * Allow for an early allocation of physically contiguous memory. 1102 */ 1103 alloc_base = contig_mem_prealloc(alloc_base, npages); 1104 1105 /* 1106 * Platforms like Starcat and OPL need special structures assigned in 1107 * 32-bit virtual address space because their probing routines execute 1108 * FCode, and FCode can't handle 64-bit virtual addresses... 1109 */ 1110 if (&plat_startup_memlist) { 1111 alloc_base = plat_startup_memlist(alloc_base); 1112 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1113 ecache_alignsize); 1114 PRM_DEBUG(alloc_base); 1115 } 1116 1117 /* 1118 * Save off where the contiguous allocations to date have ended 1119 * in econtig32. 1120 */ 1121 econtig32 = alloc_base; 1122 PRM_DEBUG(econtig32); 1123 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1124 cmn_err(CE_PANIC, "econtig32 too big"); 1125 1126 pp_sz = calc_pp_sz(npages); 1127 PRM_DEBUG(pp_sz); 1128 if (kpm_enable) { 1129 kpm_pp_sz = calc_kpmpp_sz(npages); 1130 PRM_DEBUG(kpm_pp_sz); 1131 } 1132 1133 hmehash_sz = calc_hmehash_sz(npages); 1134 PRM_DEBUG(hmehash_sz); 1135 1136 pagehash_sz = calc_pagehash_sz(npages); 1137 PRM_DEBUG(pagehash_sz); 1138 1139 pagelist_sz = calc_free_pagelist_sz(); 1140 PRM_DEBUG(pagelist_sz); 1141 1142 #ifdef TRAPTRACE 1143 tt_sz = calc_traptrace_sz(); 1144 PRM_DEBUG(tt_sz); 1145 #else 1146 tt_sz = 0; 1147 #endif /* TRAPTRACE */ 1148 1149 /* 1150 * Place the array that protects pp->p_selock in the kmem64 wad. 1151 */ 1152 pse_shift = size_pse_array(npages, max_ncpus); 1153 PRM_DEBUG(pse_shift); 1154 pse_table_size = 1 << pse_shift; 1155 PRM_DEBUG(pse_table_size); 1156 psetable_sz = roundup( 1157 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1158 PRM_DEBUG(psetable_sz); 1159 1160 /* 1161 * Now allocate the whole wad 1162 */ 1163 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1164 pagelist_sz + tt_sz + psetable_sz; 1165 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1166 kmem64_base = (caddr_t)syslimit; 1167 kmem64_end = kmem64_base + kmem64_sz; 1168 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1169 /* 1170 * Attempt for kmem64 to allocate one big 1171 * contiguous chunk of memory failed. 1172 * We get here because we are sun4v. 1173 * We will proceed by breaking up 1174 * the allocation into two attempts. 1175 * First, we allocate kpm_pp_sz, hmehash_sz, 1176 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1177 * one contiguous chunk. This is a much smaller 1178 * chunk and we should get it, if not we panic. 1179 * Note that hmehash and tt need to be physically 1180 * (in the real address sense) contiguous. 1181 * Next, we use bop_alloc_chunk() to 1182 * to allocate the page_t structures. 1183 * This will allow the page_t to be allocated 1184 * in multiple smaller chunks. 1185 * In doing so, the assumption that page_t is 1186 * physically contiguous no longer hold, this is ok 1187 * for sun4v but not for sun4u. 1188 */ 1189 size_t tmp_size; 1190 caddr_t tmp_base; 1191 1192 pp_sz = roundup(pp_sz, PAGESIZE); 1193 1194 /* 1195 * Allocate kpm_pp_sz, hmehash_sz, 1196 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1197 */ 1198 tmp_base = kmem64_base + pp_sz; 1199 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1200 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1201 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1202 prom_panic("kmem64 prom_alloc contig failed"); 1203 PRM_DEBUG(tmp_base); 1204 PRM_DEBUG(tmp_size); 1205 1206 /* 1207 * Allocate the page_ts 1208 */ 1209 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1210 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1211 PRM_DEBUG(kmem64_base); 1212 PRM_DEBUG(pp_sz); 1213 1214 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1215 ASSERT(kmem64_aligned_end >= kmem64_end); 1216 1217 kmem64_smchunks = 1; 1218 } else { 1219 1220 /* 1221 * We need to adjust pp_sz for the normal 1222 * case where kmem64 can allocate one large chunk 1223 */ 1224 if (kpm_smallpages == 0) { 1225 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1226 } else { 1227 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1228 sizeof (kpm_spage_t)); 1229 } 1230 pp_sz = npages * sizeof (struct page); 1231 } 1232 1233 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1234 cmn_err(CE_PANIC, "not enough kmem64 space"); 1235 PRM_DEBUG(kmem64_base); 1236 PRM_DEBUG(kmem64_end); 1237 PRM_DEBUG(kmem64_aligned_end); 1238 1239 /* 1240 * ... and divy it up 1241 */ 1242 alloc_base = kmem64_base; 1243 1244 pp_base = (page_t *)alloc_base; 1245 alloc_base += pp_sz; 1246 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1247 PRM_DEBUG(pp_base); 1248 PRM_DEBUG(npages); 1249 1250 if (kpm_enable) { 1251 kpm_pp_base = alloc_base; 1252 if (kpm_smallpages == 0) { 1253 /* kpm_npages based on physinstalled, don't reset */ 1254 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1255 } else { 1256 kpm_npages = ptokpmpr(npages); 1257 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1258 } 1259 alloc_base += kpm_pp_sz; 1260 alloc_base = 1261 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1262 PRM_DEBUG(kpm_pp_base); 1263 } 1264 1265 alloc_base = alloc_hmehash(alloc_base); 1266 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1267 PRM_DEBUG(alloc_base); 1268 1269 page_hash = (page_t **)alloc_base; 1270 alloc_base += pagehash_sz; 1271 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1272 PRM_DEBUG(page_hash); 1273 1274 alloc_base = alloc_page_freelists(alloc_base); 1275 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1276 PRM_DEBUG(alloc_base); 1277 1278 #ifdef TRAPTRACE 1279 ttrace_buf = alloc_base; 1280 alloc_base += tt_sz; 1281 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1282 PRM_DEBUG(alloc_base); 1283 #endif /* TRAPTRACE */ 1284 1285 pse_mutex = (pad_mutex_t *)alloc_base; 1286 alloc_base += psetable_sz; 1287 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1288 PRM_DEBUG(alloc_base); 1289 1290 /* 1291 * Note that if we use small chunk allocations for 1292 * kmem64, we need to ensure kmem64_end is the same as 1293 * kmem64_aligned_end to prevent subsequent logic from 1294 * trying to reuse the overmapping. 1295 * Otherwise we adjust kmem64_end to what we really allocated. 1296 */ 1297 if (kmem64_smchunks) { 1298 kmem64_end = kmem64_aligned_end; 1299 } else { 1300 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1301 } 1302 kmem64_sz = kmem64_end - kmem64_base; 1303 1304 if (&ecache_init_scrub_flush_area) { 1305 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1306 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1307 } 1308 1309 /* 1310 * If physmem is patched to be non-zero, use it instead of 1311 * the monitor value unless physmem is larger than the total 1312 * amount of memory on hand. 1313 */ 1314 if (physmem == 0 || physmem > npages) 1315 physmem = npages; 1316 1317 /* 1318 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1319 * is mounted as root. This memory is held down by OBP and unlike 1320 * the stub boot_archive is never released. 1321 * 1322 * In order to get things sized correctly on lower memory 1323 * machines (where the memory used by the ramdisk represents 1324 * a significant portion of memory), physmem is adjusted. 1325 * 1326 * This is done by subtracting the ramdisk_size which is set 1327 * to the size of the ramdisk (in Kb) in /etc/system at the 1328 * time the miniroot archive is constructed. 1329 */ 1330 if (root_is_ramdisk == B_TRUE) { 1331 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1332 physmem -= ramdisk_npages; 1333 } 1334 1335 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1336 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1337 1338 /* 1339 * Allocate space for the interrupt vector table. 1340 */ 1341 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1342 if (memspace != (caddr_t)intr_vec_table) 1343 prom_panic("interrupt vector table allocation failure"); 1344 1345 /* 1346 * Between now and when we finish copying in the memory lists, 1347 * allocations happen so the space gets fragmented and the 1348 * lists longer. Leave enough space for lists twice as 1349 * long as we have now; then roundup to a pagesize. 1350 */ 1351 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1352 prom_phys_avail_len() + prom_virt_avail_len()); 1353 memlist_sz *= 2; 1354 memlist_sz = roundup(memlist_sz, PAGESIZE); 1355 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1356 if (memspace == NULL) 1357 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1358 1359 memlist = (struct memlist *)memspace; 1360 memlist_end = (char *)memspace + memlist_sz; 1361 PRM_DEBUG(memlist); 1362 PRM_DEBUG(memlist_end); 1363 1364 PRM_DEBUG(sysbase); 1365 PRM_DEBUG(syslimit); 1366 kernelheap_init((void *)sysbase, (void *)syslimit, 1367 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1368 1369 /* 1370 * Take the most current snapshot we can by calling mem-update. 1371 */ 1372 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1373 &boot_physavail, &boot_physavail_len, 1374 &boot_virtavail, &boot_virtavail_len); 1375 1376 /* 1377 * Remove the space used by prom_alloc from the kernel heap 1378 * plus the area actually used by the OBP (if any) 1379 * ignoring virtual addresses in virt_avail, above syslimit. 1380 */ 1381 virt_avail = memlist; 1382 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1383 1384 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) { 1385 uint64_t range_base, range_size; 1386 1387 if ((range_base = cur->ml_address + cur->ml_size) < 1388 (uint64_t)sysbase) 1389 continue; 1390 if (range_base >= (uint64_t)syslimit) 1391 break; 1392 /* 1393 * Limit the range to end at syslimit. 1394 */ 1395 range_size = MIN(cur->ml_next->ml_address, 1396 (uint64_t)syslimit) - range_base; 1397 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1398 0, 0, (void *)range_base, (void *)(range_base + range_size), 1399 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1400 } 1401 1402 phys_avail = memlist; 1403 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1404 1405 /* 1406 * Add any extra memory at the end of the ndata region if there's at 1407 * least a page to add. There might be a few more pages available in 1408 * the middle of the ndata region, but for now they are ignored. 1409 */ 1410 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1411 if (nalloc_base == NULL) 1412 nalloc_base = nalloc_end; 1413 ndata_remain_sz = nalloc_end - nalloc_base; 1414 1415 /* 1416 * Copy physinstalled list into kernel space. 1417 */ 1418 phys_install = memlist; 1419 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1420 1421 /* 1422 * Create list of physical addrs we don't need pp's for: 1423 * kernel text 4M page 1424 * kernel data 4M page - ndata_remain_sz 1425 * kmem64 pages 1426 * 1427 * NB if adding any pages here, make sure no kpm page 1428 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1429 */ 1430 nopp_list = memlist; 1431 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1432 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1433 &memlist, &nopp_list); 1434 1435 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1436 if (!kmem64_smchunks) 1437 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1438 1439 if ((caddr_t)memlist > (memspace + memlist_sz)) 1440 prom_panic("memlist overflow"); 1441 1442 /* 1443 * Size the pcf array based on the number of cpus in the box at 1444 * boot time. 1445 */ 1446 pcf_init(); 1447 1448 /* 1449 * Initialize the page structures from the memory lists. 1450 */ 1451 kphysm_init(); 1452 1453 availrmem_initial = availrmem = freemem; 1454 PRM_DEBUG(availrmem); 1455 1456 /* 1457 * Some of the locks depend on page_hashsz being set! 1458 * kmem_init() depends on this; so, keep it here. 1459 */ 1460 page_lock_init(); 1461 1462 /* 1463 * Initialize kernel memory allocator. 1464 */ 1465 kmem_init(); 1466 1467 /* 1468 * Factor in colorequiv to check additional 'equivalent' bins 1469 */ 1470 if (&page_set_colorequiv_arr_cpu != NULL) 1471 page_set_colorequiv_arr_cpu(); 1472 else 1473 page_set_colorequiv_arr(); 1474 1475 /* 1476 * Initialize bp_mapin(). 1477 */ 1478 bp_init(shm_alignment, HAT_STRICTORDER); 1479 1480 /* 1481 * Reserve space for MPO mblock structs from the 32-bit heap. 1482 */ 1483 1484 if (mpo_heap32_bufsz > (size_t)0) { 1485 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1486 PAGESIZE, 0, 0, mpo_heap32_buf, 1487 mpo_heap32_buf + mpo_heap32_bufsz, 1488 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1489 } 1490 mem_config_init(); 1491 } 1492 1493 static void 1494 startup_modules(void) 1495 { 1496 int nhblk1, nhblk8; 1497 size_t nhblksz; 1498 pgcnt_t pages_per_hblk; 1499 size_t hme8blk_sz, hme1blk_sz; 1500 1501 /* 1502 * The system file /etc/system was read already under startup_memlist. 1503 */ 1504 if (&set_platform_defaults) 1505 set_platform_defaults(); 1506 1507 /* 1508 * Calculate default settings of system parameters based upon 1509 * maxusers, yet allow to be overridden via the /etc/system file. 1510 */ 1511 param_calc(0); 1512 1513 mod_setup(); 1514 1515 /* 1516 * Initialize system parameters 1517 */ 1518 param_init(); 1519 1520 /* 1521 * maxmem is the amount of physical memory we're playing with. 1522 */ 1523 maxmem = physmem; 1524 1525 /* Set segkp limits. */ 1526 ncbase = kdi_segdebugbase; 1527 ncend = kdi_segdebugbase; 1528 1529 /* 1530 * Initialize the hat layer. 1531 */ 1532 hat_init(); 1533 1534 /* 1535 * Initialize segment management stuff. 1536 */ 1537 seg_init(); 1538 1539 /* 1540 * Create the va>tte handler, so the prom can understand 1541 * kernel translations. The handler is installed later, just 1542 * as we are about to take over the trap table from the prom. 1543 */ 1544 create_va_to_tte(); 1545 1546 /* 1547 * Load the forthdebugger (optional) 1548 */ 1549 forthdebug_init(); 1550 1551 /* 1552 * Create OBP node for console input callbacks 1553 * if it is needed. 1554 */ 1555 startup_create_io_node(); 1556 1557 if (modloadonly("fs", "specfs") == -1) 1558 halt("Can't load specfs"); 1559 1560 if (modloadonly("fs", "devfs") == -1) 1561 halt("Can't load devfs"); 1562 1563 if (modloadonly("fs", "procfs") == -1) 1564 halt("Can't load procfs"); 1565 1566 if (modloadonly("misc", "swapgeneric") == -1) 1567 halt("Can't load swapgeneric"); 1568 1569 (void) modloadonly("sys", "lbl_edition"); 1570 1571 dispinit(); 1572 1573 /* 1574 * Infer meanings to the members of the idprom buffer. 1575 */ 1576 parse_idprom(); 1577 1578 /* Read cluster configuration data. */ 1579 clconf_init(); 1580 1581 setup_ddi(); 1582 1583 /* 1584 * Lets take this opportunity to load the root device. 1585 */ 1586 if (loadrootmodules() != 0) 1587 debug_enter("Can't load the root filesystem"); 1588 1589 /* 1590 * Load tod driver module for the tod part found on this system. 1591 * Recompute the cpu frequency/delays based on tod as tod part 1592 * tends to keep time more accurately. 1593 */ 1594 if (&load_tod_module) 1595 load_tod_module(); 1596 1597 /* 1598 * Allow platforms to load modules which might 1599 * be needed after bootops are gone. 1600 */ 1601 if (&load_platform_modules) 1602 load_platform_modules(); 1603 1604 setcpudelay(); 1605 1606 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1607 &boot_physavail, &boot_physavail_len, 1608 &boot_virtavail, &boot_virtavail_len); 1609 1610 /* 1611 * Calculation and allocation of hmeblks needed to remap 1612 * the memory allocated by PROM till now. 1613 * Overestimate the number of hblk1 elements by assuming 1614 * worst case of TTE64K mappings. 1615 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1616 */ 1617 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1618 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1619 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1620 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1621 1622 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1623 1624 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1625 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1626 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1627 1628 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1629 1630 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1631 nhblk8 = 0; 1632 while (bop_alloc_pages > 1) { 1633 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1634 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1635 bop_alloc_pages *= hme8blk_sz; 1636 bop_alloc_pages = btopr(bop_alloc_pages); 1637 } 1638 nhblk8 += 2; 1639 1640 /* 1641 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1642 * boundary, the number of hblk8's needed to map the entries in the 1643 * boot_virtavail list needs to be adjusted to take this into 1644 * consideration. Thus, we need to add additional hblk8's since it 1645 * is possible that an hblk8 will not have all 8 slots used due to 1646 * alignment constraints. Since there were boot_virtavail_len entries 1647 * in that list, we need to add that many hblk8's to the number 1648 * already calculated to make sure we don't underestimate. 1649 */ 1650 nhblk8 += boot_virtavail_len; 1651 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1652 1653 /* Allocate in pagesize chunks */ 1654 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1655 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1656 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1657 } 1658 1659 static void 1660 startup_bop_gone(void) 1661 { 1662 1663 /* 1664 * Destroy the MD initialized at startup 1665 * The startup initializes the MD framework 1666 * using prom and BOP alloc free it now. 1667 */ 1668 mach_descrip_startup_fini(); 1669 1670 /* 1671 * We're done with prom allocations. 1672 */ 1673 bop_fini(); 1674 1675 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1676 &boot_physavail, &boot_physavail_len, 1677 &boot_virtavail, &boot_virtavail_len); 1678 1679 /* 1680 * setup physically contiguous area twice as large as the ecache. 1681 * this is used while doing displacement flush of ecaches 1682 */ 1683 if (&ecache_flush_address) { 1684 ecache_flushaddr = ecache_flush_address(); 1685 if (ecache_flushaddr == (uint64_t)-1) { 1686 cmn_err(CE_PANIC, 1687 "startup: no memory to set ecache_flushaddr"); 1688 } 1689 } 1690 1691 /* 1692 * Virtual available next. 1693 */ 1694 ASSERT(virt_avail != NULL); 1695 memlist_free_list(virt_avail); 1696 virt_avail = memlist; 1697 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1698 1699 } 1700 1701 1702 /* 1703 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1704 * allocations have been performed. We can't call it in startup_bop_gone 1705 * since later operations can cause obp to allocate more memory. 1706 */ 1707 void 1708 startup_fixup_physavail(void) 1709 { 1710 struct memlist *cur; 1711 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1712 1713 PRM_DEBUG(kmem64_overmap_size); 1714 1715 /* 1716 * take the most current snapshot we can by calling mem-update 1717 */ 1718 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1719 &boot_physavail, &boot_physavail_len, 1720 &boot_virtavail, &boot_virtavail_len); 1721 1722 /* 1723 * Copy phys_avail list, again. 1724 * Both the kernel/boot and the prom have been allocating 1725 * from the original list we copied earlier. 1726 */ 1727 cur = memlist; 1728 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1729 1730 /* 1731 * Add any unused kmem64 memory from overmapped page 1732 * (Note: va_to_pa does not work for kmem64_end) 1733 */ 1734 if (kmem64_overmap_size) { 1735 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1736 kmem64_overmap_size, &memlist, &cur); 1737 } 1738 1739 /* 1740 * Add any extra memory after e_data we added to the phys_avail list 1741 * back to the old list. 1742 */ 1743 if (ndata_remain_sz >= MMU_PAGESIZE) 1744 memlist_add(va_to_pa(nalloc_base), 1745 (uint64_t)ndata_remain_sz, &memlist, &cur); 1746 1747 /* 1748 * There isn't any bounds checking on the memlist area 1749 * so ensure it hasn't overgrown. 1750 */ 1751 if ((caddr_t)memlist > (caddr_t)memlist_end) 1752 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1753 1754 /* 1755 * The kernel removes the pages that were allocated for it from 1756 * the freelist, but we now have to find any -extra- pages that 1757 * the prom has allocated for it's own book-keeping, and remove 1758 * them from the freelist too. sigh. 1759 */ 1760 sync_memlists(phys_avail, cur); 1761 1762 ASSERT(phys_avail != NULL); 1763 1764 old_phys_avail = phys_avail; 1765 phys_avail = cur; 1766 } 1767 1768 void 1769 update_kcage_ranges(uint64_t addr, uint64_t len) 1770 { 1771 pfn_t base = btop(addr); 1772 pgcnt_t num = btop(len); 1773 int rv; 1774 1775 rv = kcage_range_add(base, num, kcage_startup_dir); 1776 1777 if (rv == ENOMEM) { 1778 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1779 (len == 0 ? 0 : BYTES_TO_MB(len))); 1780 } else if (rv != 0) { 1781 /* catch this in debug kernels */ 1782 ASSERT(0); 1783 1784 cmn_err(CE_WARN, "unexpected kcage_range_add" 1785 " return value %d", rv); 1786 } 1787 } 1788 1789 static void 1790 startup_vm(void) 1791 { 1792 size_t i; 1793 struct segmap_crargs a; 1794 struct segkpm_crargs b; 1795 1796 uint64_t avmem; 1797 caddr_t va; 1798 pgcnt_t max_phys_segkp; 1799 int mnode; 1800 1801 extern int use_brk_lpg, use_stk_lpg; 1802 1803 /* 1804 * get prom's mappings, create hments for them and switch 1805 * to the kernel context. 1806 */ 1807 hat_kern_setup(); 1808 1809 /* 1810 * Take over trap table 1811 */ 1812 setup_trap_table(); 1813 1814 /* 1815 * Install the va>tte handler, so that the prom can handle 1816 * misses and understand the kernel table layout in case 1817 * we need call into the prom. 1818 */ 1819 install_va_to_tte(); 1820 1821 /* 1822 * Set a flag to indicate that the tba has been taken over. 1823 */ 1824 tba_taken_over = 1; 1825 1826 /* initialize MMU primary context register */ 1827 mmu_init_kcontext(); 1828 1829 /* 1830 * The boot cpu can now take interrupts, x-calls, x-traps 1831 */ 1832 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1833 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1834 1835 /* 1836 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1837 */ 1838 tbr_wr_addr_inited = 1; 1839 1840 /* 1841 * Initialize VM system, and map kernel address space. 1842 */ 1843 kvm_init(); 1844 1845 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1846 if (kernel_cage_enable) { 1847 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1848 } 1849 memlist_free_list(old_phys_avail); 1850 1851 /* 1852 * If the following is true, someone has patched 1853 * phsymem to be less than the number of pages that 1854 * the system actually has. Remove pages until system 1855 * memory is limited to the requested amount. Since we 1856 * have allocated page structures for all pages, we 1857 * correct the amount of memory we want to remove 1858 * by the size of the memory used to hold page structures 1859 * for the non-used pages. 1860 */ 1861 if (physmem + ramdisk_npages < npages) { 1862 pgcnt_t diff, off; 1863 struct page *pp; 1864 struct seg kseg; 1865 1866 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1867 1868 off = 0; 1869 diff = npages - (physmem + ramdisk_npages); 1870 diff -= mmu_btopr(diff * sizeof (struct page)); 1871 kseg.s_as = &kas; 1872 while (diff--) { 1873 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1874 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1875 &kseg, (caddr_t)off); 1876 if (pp == NULL) 1877 cmn_err(CE_PANIC, "limited physmem too much!"); 1878 page_io_unlock(pp); 1879 page_downgrade(pp); 1880 availrmem--; 1881 off += MMU_PAGESIZE; 1882 } 1883 } 1884 1885 /* 1886 * When printing memory, show the total as physmem less 1887 * that stolen by a debugger. 1888 */ 1889 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1890 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1891 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1892 1893 avmem = (uint64_t)freemem << PAGESHIFT; 1894 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1895 1896 /* 1897 * For small memory systems disable automatic large pages. 1898 */ 1899 if (physmem < privm_lpg_min_physmem) { 1900 use_brk_lpg = 0; 1901 use_stk_lpg = 0; 1902 } 1903 1904 /* 1905 * Perform platform specific freelist processing 1906 */ 1907 if (&plat_freelist_process) { 1908 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1909 if (mem_node_config[mnode].exists) 1910 plat_freelist_process(mnode); 1911 } 1912 1913 /* 1914 * Initialize the segkp segment type. We position it 1915 * after the configured tables and buffers (whose end 1916 * is given by econtig) and before V_WKBASE_ADDR. 1917 * Also in this area is segkmap (size SEGMAPSIZE). 1918 */ 1919 1920 /* XXX - cache alignment? */ 1921 va = (caddr_t)SEGKPBASE; 1922 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1923 1924 max_phys_segkp = (physmem * 2); 1925 1926 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1927 segkpsize = btop(SEGKPDEFSIZE); 1928 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1929 "segkpsize has been reset to %ld pages", segkpsize); 1930 } 1931 1932 i = ptob(MIN(segkpsize, max_phys_segkp)); 1933 1934 rw_enter(&kas.a_lock, RW_WRITER); 1935 if (seg_attach(&kas, va, i, segkp) < 0) 1936 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1937 if (segkp_create(segkp) != 0) 1938 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1939 rw_exit(&kas.a_lock); 1940 1941 /* 1942 * kpm segment 1943 */ 1944 segmap_kpm = kpm_enable && 1945 segmap_kpm && PAGESIZE == MAXBSIZE; 1946 1947 if (kpm_enable) { 1948 rw_enter(&kas.a_lock, RW_WRITER); 1949 1950 /* 1951 * The segkpm virtual range range is larger than the 1952 * actual physical memory size and also covers gaps in 1953 * the physical address range for the following reasons: 1954 * . keep conversion between segkpm and physical addresses 1955 * simple, cheap and unambiguous. 1956 * . avoid extension/shrink of the the segkpm in case of DR. 1957 * . avoid complexity for handling of virtual addressed 1958 * caches, segkpm and the regular mapping scheme must be 1959 * kept in sync wrt. the virtual color of mapped pages. 1960 * Any accesses to virtual segkpm ranges not backed by 1961 * physical memory will fall through the memseg pfn hash 1962 * and will be handled in segkpm_fault. 1963 * Additional kpm_size spaces needed for vac alias prevention. 1964 */ 1965 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1966 segkpm) < 0) 1967 cmn_err(CE_PANIC, "cannot attach segkpm"); 1968 1969 b.prot = PROT_READ | PROT_WRITE; 1970 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1971 1972 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1973 panic("segkpm_create segkpm"); 1974 1975 rw_exit(&kas.a_lock); 1976 1977 mach_kpm_init(); 1978 } 1979 1980 va = kpm_vbase + (kpm_size * vac_colors); 1981 1982 if (!segzio_fromheap) { 1983 size_t size; 1984 size_t physmem_b = mmu_ptob(physmem); 1985 1986 /* size is in bytes, segziosize is in pages */ 1987 if (segziosize == 0) { 1988 size = physmem_b; 1989 } else { 1990 size = mmu_ptob(segziosize); 1991 } 1992 1993 if (size < SEGZIOMINSIZE) { 1994 size = SEGZIOMINSIZE; 1995 } else if (size > SEGZIOMAXSIZE) { 1996 size = SEGZIOMAXSIZE; 1997 /* 1998 * On 64-bit x86, we only have 2TB of KVA. This exists 1999 * for parity with x86. 2000 * 2001 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2002 * doesn't consume all of KVA. However, if we have a 2003 * system that has more thant 512gb of physical memory, 2004 * we can actually consume about half of the difference 2005 * between 512gb and the rest of the available physical 2006 * memory. 2007 */ 2008 if (physmem_b > SEGZIOMAXSIZE) { 2009 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2010 } 2011 } 2012 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2013 /* put the base of the ZIO segment after the kpm segment */ 2014 segzio_base = va; 2015 va += mmu_ptob(segziosize); 2016 PRM_DEBUG(segziosize); 2017 PRM_DEBUG(segzio_base); 2018 2019 /* 2020 * On some platforms, kvm_init is called after the kpm 2021 * sizes have been determined. On SPARC, kvm_init is called 2022 * before, so we have to attach the kzioseg after kvm is 2023 * initialized, otherwise we'll try to allocate from the boot 2024 * area since the kernel heap hasn't yet been configured. 2025 */ 2026 rw_enter(&kas.a_lock, RW_WRITER); 2027 2028 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2029 &kzioseg); 2030 (void) segkmem_create(&kzioseg); 2031 2032 /* create zio area covering new segment */ 2033 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2034 2035 rw_exit(&kas.a_lock); 2036 } 2037 2038 if (ppvm_enable) { 2039 caddr_t ppvm_max; 2040 2041 /* 2042 * ppvm refers to the static VA space used to map 2043 * the page_t's for dynamically added memory. 2044 * 2045 * ppvm_base should not cross a potential VA hole. 2046 * 2047 * ppvm_size should be large enough to map the 2048 * page_t's needed to manage all of KPM range. 2049 */ 2050 ppvm_size = 2051 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2052 MMU_PAGESIZE); 2053 ppvm_max = (caddr_t)(0ull - ppvm_size); 2054 ppvm_base = (page_t *)va; 2055 2056 if ((caddr_t)ppvm_base <= hole_end) { 2057 cmn_err(CE_WARN, 2058 "Memory DR disabled: invalid DR map base: 0x%p\n", 2059 (void *)ppvm_base); 2060 ppvm_enable = 0; 2061 } else if ((caddr_t)ppvm_base > ppvm_max) { 2062 uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2063 2064 cmn_err(CE_WARN, 2065 "Memory DR disabled: insufficient DR map size:" 2066 " 0x%lx (needed 0x%lx)\n", 2067 ppvm_size - diff, ppvm_size); 2068 ppvm_enable = 0; 2069 } 2070 PRM_DEBUG(ppvm_size); 2071 PRM_DEBUG(ppvm_base); 2072 } 2073 2074 /* 2075 * Now create generic mapping segment. This mapping 2076 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2077 * virtual address is greater than the amount of free 2078 * memory that is available, then we trim back the 2079 * segment size to that amount 2080 */ 2081 va = (caddr_t)SEGMAPBASE; 2082 2083 /* 2084 * 1201049: segkmap base address must be MAXBSIZE aligned 2085 */ 2086 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2087 2088 /* 2089 * Set size of segmap to percentage of freemem at boot, 2090 * but stay within the allowable range 2091 * Note we take percentage before converting from pages 2092 * to bytes to avoid an overflow on 32-bit kernels. 2093 */ 2094 i = mmu_ptob((freemem * segmap_percent) / 100); 2095 2096 if (i < MINMAPSIZE) 2097 i = MINMAPSIZE; 2098 2099 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2100 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2101 2102 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2103 2104 rw_enter(&kas.a_lock, RW_WRITER); 2105 if (seg_attach(&kas, va, i, segkmap) < 0) 2106 cmn_err(CE_PANIC, "cannot attach segkmap"); 2107 2108 a.prot = PROT_READ | PROT_WRITE; 2109 a.shmsize = shm_alignment; 2110 a.nfreelist = 0; /* use segmap driver defaults */ 2111 2112 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2113 panic("segmap_create segkmap"); 2114 rw_exit(&kas.a_lock); 2115 2116 segdev_init(); 2117 } 2118 2119 static void 2120 startup_end(void) 2121 { 2122 if ((caddr_t)memlist > (caddr_t)memlist_end) 2123 panic("memlist overflow 2"); 2124 memlist_free_block((caddr_t)memlist, 2125 ((caddr_t)memlist_end - (caddr_t)memlist)); 2126 memlist = NULL; 2127 2128 /* enable page_relocation since OBP is now done */ 2129 page_relocate_ready = 1; 2130 2131 /* 2132 * Perform tasks that get done after most of the VM 2133 * initialization has been done but before the clock 2134 * and other devices get started. 2135 */ 2136 kern_setup1(); 2137 2138 /* 2139 * Perform CPC initialization for this CPU. 2140 */ 2141 kcpc_hw_init(); 2142 2143 /* 2144 * Intialize the VM arenas for allocating physically 2145 * contiguus memory chunk for interrupt queues snd 2146 * allocate/register boot cpu's queues, if any and 2147 * allocate dump buffer for sun4v systems to store 2148 * extra crash information during crash dump 2149 */ 2150 contig_mem_init(); 2151 mach_descrip_init(); 2152 2153 if (cpu_intrq_setup(CPU)) { 2154 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2155 } 2156 cpu_intrq_register(CPU); 2157 mach_htraptrace_setup(CPU->cpu_id); 2158 mach_htraptrace_configure(CPU->cpu_id); 2159 mach_dump_buffer_init(); 2160 2161 /* 2162 * Initialize interrupt related stuff 2163 */ 2164 cpu_intr_alloc(CPU, NINTR_THREADS); 2165 2166 (void) splzs(); /* allow hi clock ints but not zs */ 2167 2168 /* 2169 * Initialize errors. 2170 */ 2171 error_init(); 2172 2173 /* 2174 * Note that we may have already used kernel bcopy before this 2175 * point - but if you really care about this, adb the use_hw_* 2176 * variables to 0 before rebooting. 2177 */ 2178 mach_hw_copy_limit(); 2179 2180 /* 2181 * Install the "real" preemption guards before DDI services 2182 * are available. 2183 */ 2184 (void) prom_set_preprom(kern_preprom); 2185 (void) prom_set_postprom(kern_postprom); 2186 CPU->cpu_m.mutex_ready = 1; 2187 2188 /* 2189 * Initialize segnf (kernel support for non-faulting loads). 2190 */ 2191 segnf_init(); 2192 2193 /* 2194 * Configure the root devinfo node. 2195 */ 2196 configure(); /* set up devices */ 2197 mach_cpu_halt_idle(); 2198 } 2199 2200 2201 void 2202 post_startup(void) 2203 { 2204 #ifdef PTL1_PANIC_DEBUG 2205 extern void init_ptl1_thread(void); 2206 #endif /* PTL1_PANIC_DEBUG */ 2207 extern void abort_sequence_init(void); 2208 2209 /* 2210 * Set the system wide, processor-specific flags to be passed 2211 * to userland via the aux vector for performance hints and 2212 * instruction set extensions. 2213 */ 2214 bind_hwcap(); 2215 2216 /* 2217 * Startup memory scrubber (if any) 2218 */ 2219 mach_memscrub(); 2220 2221 /* 2222 * Allocate soft interrupt to handle abort sequence. 2223 */ 2224 abort_sequence_init(); 2225 2226 /* 2227 * Configure the rest of the system. 2228 * Perform forceloading tasks for /etc/system. 2229 */ 2230 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2231 /* 2232 * ON4.0: Force /proc module in until clock interrupt handle fixed 2233 * ON4.0: This must be fixed or restated in /etc/systems. 2234 */ 2235 (void) modload("fs", "procfs"); 2236 2237 /* load machine class specific drivers */ 2238 load_mach_drivers(); 2239 2240 /* load platform specific drivers */ 2241 if (&load_platform_drivers) 2242 load_platform_drivers(); 2243 2244 /* load vis simulation module, if we are running w/fpu off */ 2245 if (!fpu_exists) { 2246 if (modload("misc", "vis") == -1) 2247 halt("Can't load vis"); 2248 } 2249 2250 mach_fpras(); 2251 2252 maxmem = freemem; 2253 2254 pg_init(); 2255 2256 #ifdef PTL1_PANIC_DEBUG 2257 init_ptl1_thread(); 2258 #endif /* PTL1_PANIC_DEBUG */ 2259 } 2260 2261 #ifdef PTL1_PANIC_DEBUG 2262 int ptl1_panic_test = 0; 2263 int ptl1_panic_xc_one_test = 0; 2264 int ptl1_panic_xc_all_test = 0; 2265 int ptl1_panic_xt_one_test = 0; 2266 int ptl1_panic_xt_all_test = 0; 2267 kthread_id_t ptl1_thread_p = NULL; 2268 kcondvar_t ptl1_cv; 2269 kmutex_t ptl1_mutex; 2270 int ptl1_recurse_count_threshold = 0x40; 2271 int ptl1_recurse_trap_threshold = 0x3d; 2272 extern void ptl1_recurse(int, int); 2273 extern void ptl1_panic_xt(int, int); 2274 2275 /* 2276 * Called once per second by timeout() to wake up 2277 * the ptl1_panic thread to see if it should cause 2278 * a trap to the ptl1_panic() code. 2279 */ 2280 /* ARGSUSED */ 2281 static void 2282 ptl1_wakeup(void *arg) 2283 { 2284 mutex_enter(&ptl1_mutex); 2285 cv_signal(&ptl1_cv); 2286 mutex_exit(&ptl1_mutex); 2287 } 2288 2289 /* 2290 * ptl1_panic cross call function: 2291 * Needed because xc_one() and xc_some() can pass 2292 * 64 bit args but ptl1_recurse() expects ints. 2293 */ 2294 static void 2295 ptl1_panic_xc(void) 2296 { 2297 ptl1_recurse(ptl1_recurse_count_threshold, 2298 ptl1_recurse_trap_threshold); 2299 } 2300 2301 /* 2302 * The ptl1 thread waits for a global flag to be set 2303 * and uses the recurse thresholds to set the stack depth 2304 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2305 * or indirectly via the cross call and cross trap functions. 2306 * 2307 * This is useful testing stack overflows and normal 2308 * ptl1_panic() states with a know stack frame. 2309 * 2310 * ptl1_recurse() is an asm function in ptl1_panic.s that 2311 * sets the {In, Local, Out, and Global} registers to a 2312 * know state on the stack and just prior to causing a 2313 * test ptl1_panic trap. 2314 */ 2315 static void 2316 ptl1_thread(void) 2317 { 2318 mutex_enter(&ptl1_mutex); 2319 while (ptl1_thread_p) { 2320 cpuset_t other_cpus; 2321 int cpu_id; 2322 int my_cpu_id; 2323 int target_cpu_id; 2324 int target_found; 2325 2326 if (ptl1_panic_test) { 2327 ptl1_recurse(ptl1_recurse_count_threshold, 2328 ptl1_recurse_trap_threshold); 2329 } 2330 2331 /* 2332 * Find potential targets for x-call and x-trap, 2333 * if any exist while preempt is disabled we 2334 * start a ptl1_panic if requested via a 2335 * globals. 2336 */ 2337 kpreempt_disable(); 2338 my_cpu_id = CPU->cpu_id; 2339 other_cpus = cpu_ready_set; 2340 CPUSET_DEL(other_cpus, CPU->cpu_id); 2341 target_found = 0; 2342 if (!CPUSET_ISNULL(other_cpus)) { 2343 /* 2344 * Pick the first one 2345 */ 2346 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2347 if (cpu_id == my_cpu_id) 2348 continue; 2349 2350 if (CPU_XCALL_READY(cpu_id)) { 2351 target_cpu_id = cpu_id; 2352 target_found = 1; 2353 break; 2354 } 2355 } 2356 ASSERT(target_found); 2357 2358 if (ptl1_panic_xc_one_test) { 2359 xc_one(target_cpu_id, 2360 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2361 } 2362 if (ptl1_panic_xc_all_test) { 2363 xc_some(other_cpus, 2364 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2365 } 2366 if (ptl1_panic_xt_one_test) { 2367 xt_one(target_cpu_id, 2368 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2369 } 2370 if (ptl1_panic_xt_all_test) { 2371 xt_some(other_cpus, 2372 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2373 } 2374 } 2375 kpreempt_enable(); 2376 (void) timeout(ptl1_wakeup, NULL, hz); 2377 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2378 } 2379 mutex_exit(&ptl1_mutex); 2380 } 2381 2382 /* 2383 * Called during early startup to create the ptl1_thread 2384 */ 2385 void 2386 init_ptl1_thread(void) 2387 { 2388 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2389 &p0, TS_RUN, 0); 2390 } 2391 #endif /* PTL1_PANIC_DEBUG */ 2392 2393 2394 static void 2395 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2396 { 2397 struct memlist *new; 2398 2399 new = *memlistp; 2400 new->ml_address = start; 2401 new->ml_size = len; 2402 *memlistp = new + 1; 2403 } 2404 2405 /* 2406 * Add to a memory list. 2407 * start = start of new memory segment 2408 * len = length of new memory segment in bytes 2409 * memlistp = pointer to array of available memory segment structures 2410 * curmemlistp = memory list to which to add segment. 2411 */ 2412 static void 2413 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2414 struct memlist **curmemlistp) 2415 { 2416 struct memlist *new = *memlistp; 2417 2418 memlist_new(start, len, memlistp); 2419 memlist_insert(new, curmemlistp); 2420 } 2421 2422 static int 2423 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2424 { 2425 int nseg; 2426 size_t memseg_sz; 2427 struct memseg *msp; 2428 2429 /* 2430 * The memseg list is for the chunks of physical memory that 2431 * will be managed by the vm system. The number calculated is 2432 * a guess as boot may fragment it more when memory allocations 2433 * are made before kphysm_init(). 2434 */ 2435 memseg_sz = (avail + 10) * sizeof (struct memseg); 2436 memseg_sz = roundup(memseg_sz, PAGESIZE); 2437 nseg = memseg_sz / sizeof (struct memseg); 2438 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2439 if (msp == NULL) 2440 return (1); 2441 PRM_DEBUG(memseg_free); 2442 2443 while (nseg--) { 2444 msp->next = memseg_free; 2445 memseg_free = msp; 2446 msp++; 2447 } 2448 return (0); 2449 } 2450 2451 /* 2452 * In the case of architectures that support dynamic addition of 2453 * memory at run-time there are two cases where memsegs need to 2454 * be initialized and added to the memseg list. 2455 * 1) memsegs that are constructed at startup. 2456 * 2) memsegs that are constructed at run-time on 2457 * hot-plug capable architectures. 2458 * This code was originally part of the function kphysm_init(). 2459 */ 2460 2461 static void 2462 memseg_list_add(struct memseg *memsegp) 2463 { 2464 struct memseg **prev_memsegp; 2465 pgcnt_t num; 2466 2467 /* insert in memseg list, decreasing number of pages order */ 2468 2469 num = MSEG_NPAGES(memsegp); 2470 2471 for (prev_memsegp = &memsegs; *prev_memsegp; 2472 prev_memsegp = &((*prev_memsegp)->next)) { 2473 if (num > MSEG_NPAGES(*prev_memsegp)) 2474 break; 2475 } 2476 2477 memsegp->next = *prev_memsegp; 2478 *prev_memsegp = memsegp; 2479 2480 if (kpm_enable) { 2481 memsegp->nextpa = (memsegp->next) ? 2482 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2483 2484 if (prev_memsegp != &memsegs) { 2485 struct memseg *msp; 2486 msp = (struct memseg *)((caddr_t)prev_memsegp - 2487 offsetof(struct memseg, next)); 2488 msp->nextpa = va_to_pa(memsegp); 2489 } else { 2490 memsegspa = va_to_pa(memsegs); 2491 } 2492 } 2493 } 2494 2495 /* 2496 * PSM add_physmem_cb(). US-II and newer processors have some 2497 * flavor of the prefetch capability implemented. We exploit 2498 * this capability for optimum performance. 2499 */ 2500 #define PREFETCH_BYTES 64 2501 2502 void 2503 add_physmem_cb(page_t *pp, pfn_t pnum) 2504 { 2505 extern void prefetch_page_w(void *); 2506 2507 pp->p_pagenum = pnum; 2508 2509 /* 2510 * Prefetch one more page_t into E$. To prevent future 2511 * mishaps with the sizeof(page_t) changing on us, we 2512 * catch this on debug kernels if we can't bring in the 2513 * entire hpage with 2 PREFETCH_BYTES reads. See 2514 * also, sun4u/cpu/cpu_module.c 2515 */ 2516 /*LINTED*/ 2517 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2518 prefetch_page_w((char *)pp); 2519 } 2520 2521 /* 2522 * Find memseg with given pfn 2523 */ 2524 static struct memseg * 2525 memseg_find(pfn_t base, pfn_t *next) 2526 { 2527 struct memseg *seg; 2528 2529 if (next != NULL) 2530 *next = LONG_MAX; 2531 for (seg = memsegs; seg != NULL; seg = seg->next) { 2532 if (base >= seg->pages_base && base < seg->pages_end) 2533 return (seg); 2534 if (next != NULL && seg->pages_base > base && 2535 seg->pages_base < *next) 2536 *next = seg->pages_base; 2537 } 2538 return (NULL); 2539 } 2540 2541 /* 2542 * Put page allocated by OBP on prom_ppages 2543 */ 2544 static void 2545 kphysm_erase(uint64_t addr, uint64_t len) 2546 { 2547 struct page *pp; 2548 struct memseg *seg; 2549 pfn_t base = btop(addr), next; 2550 pgcnt_t num = btop(len); 2551 2552 while (num != 0) { 2553 pgcnt_t off, left; 2554 2555 seg = memseg_find(base, &next); 2556 if (seg == NULL) { 2557 if (next == LONG_MAX) 2558 break; 2559 left = MIN(next - base, num); 2560 base += left, num -= left; 2561 continue; 2562 } 2563 off = base - seg->pages_base; 2564 pp = seg->pages + off; 2565 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2566 while (num != left) { 2567 /* 2568 * init it, lock it, and hashin on prom_pages vp. 2569 * 2570 * Mark it as NONRELOC to let DR know the page 2571 * is locked long term, otherwise DR hangs when 2572 * trying to remove those pages. 2573 * 2574 * XXX vnode offsets on the prom_ppages vnode 2575 * are page numbers (gack) for >32 bit 2576 * physical memory machines. 2577 */ 2578 PP_SETNORELOC(pp); 2579 add_physmem_cb(pp, base); 2580 if (page_trylock(pp, SE_EXCL) == 0) 2581 cmn_err(CE_PANIC, "prom page locked"); 2582 (void) page_hashin(pp, &promvp, 2583 (offset_t)base, NULL); 2584 (void) page_pp_lock(pp, 0, 1); 2585 pp++, base++, num--; 2586 } 2587 } 2588 } 2589 2590 static page_t *ppnext; 2591 static pgcnt_t ppleft; 2592 2593 static void *kpm_ppnext; 2594 static pgcnt_t kpm_ppleft; 2595 2596 /* 2597 * Create a memseg 2598 */ 2599 static void 2600 kphysm_memseg(uint64_t addr, uint64_t len) 2601 { 2602 pfn_t base = btop(addr); 2603 pgcnt_t num = btop(len); 2604 struct memseg *seg; 2605 2606 seg = memseg_free; 2607 memseg_free = seg->next; 2608 ASSERT(seg != NULL); 2609 2610 seg->pages = ppnext; 2611 seg->epages = ppnext + num; 2612 seg->pages_base = base; 2613 seg->pages_end = base + num; 2614 ppnext += num; 2615 ppleft -= num; 2616 2617 if (kpm_enable) { 2618 pgcnt_t kpnum = ptokpmpr(num); 2619 2620 if (kpnum > kpm_ppleft) 2621 panic("kphysm_memseg: kpm_pp overflow"); 2622 seg->pagespa = va_to_pa(seg->pages); 2623 seg->epagespa = va_to_pa(seg->epages); 2624 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2625 seg->kpm_nkpmpgs = kpnum; 2626 /* 2627 * In the kpm_smallpage case, the kpm array 2628 * is 1-1 wrt the page array 2629 */ 2630 if (kpm_smallpages) { 2631 kpm_spage_t *kpm_pp = kpm_ppnext; 2632 2633 kpm_ppnext = kpm_pp + kpnum; 2634 seg->kpm_spages = kpm_pp; 2635 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2636 } else { 2637 kpm_page_t *kpm_pp = kpm_ppnext; 2638 2639 kpm_ppnext = kpm_pp + kpnum; 2640 seg->kpm_pages = kpm_pp; 2641 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2642 /* ASSERT no kpm overlaps */ 2643 ASSERT( 2644 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2645 ASSERT(memseg_find( 2646 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2647 } 2648 kpm_ppleft -= kpnum; 2649 } 2650 2651 memseg_list_add(seg); 2652 } 2653 2654 /* 2655 * Add range to free list 2656 */ 2657 void 2658 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2659 { 2660 struct page *pp; 2661 struct memseg *seg; 2662 pfn_t base = btop(addr); 2663 pgcnt_t num = btop(len); 2664 2665 seg = memseg_find(base, NULL); 2666 ASSERT(seg != NULL); 2667 pp = seg->pages + (base - seg->pages_base); 2668 2669 if (reclaim) { 2670 struct page *rpp = pp; 2671 struct page *lpp = pp + num; 2672 2673 /* 2674 * page should be locked on prom_ppages 2675 * unhash and unlock it 2676 */ 2677 while (rpp < lpp) { 2678 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp); 2679 ASSERT(PP_ISNORELOC(rpp)); 2680 PP_CLRNORELOC(rpp); 2681 page_pp_unlock(rpp, 0, 1); 2682 page_hashout(rpp, NULL); 2683 page_unlock(rpp); 2684 rpp++; 2685 } 2686 } 2687 2688 /* 2689 * add_physmem() initializes the PSM part of the page 2690 * struct by calling the PSM back with add_physmem_cb(). 2691 * In addition it coalesces pages into larger pages as 2692 * it initializes them. 2693 */ 2694 add_physmem(pp, num, base); 2695 } 2696 2697 /* 2698 * kphysm_init() tackles the problem of initializing physical memory. 2699 */ 2700 static void 2701 kphysm_init(void) 2702 { 2703 struct memlist *pmem; 2704 2705 ASSERT(page_hash != NULL && page_hashsz != 0); 2706 2707 ppnext = pp_base; 2708 ppleft = npages; 2709 kpm_ppnext = kpm_pp_base; 2710 kpm_ppleft = kpm_npages; 2711 2712 /* 2713 * installed pages not on nopp_memlist go in memseg list 2714 */ 2715 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2716 2717 /* 2718 * Free the avail list 2719 */ 2720 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next) 2721 kphysm_add(pmem->ml_address, pmem->ml_size, 0); 2722 2723 /* 2724 * Erase pages that aren't available 2725 */ 2726 diff_memlists(phys_install, phys_avail, kphysm_erase); 2727 2728 build_pfn_hash(); 2729 } 2730 2731 /* 2732 * Kernel VM initialization. 2733 * Assumptions about kernel address space ordering: 2734 * (1) gap (user space) 2735 * (2) kernel text 2736 * (3) kernel data/bss 2737 * (4) gap 2738 * (5) kernel data structures 2739 * (6) gap 2740 * (7) debugger (optional) 2741 * (8) monitor 2742 * (9) gap (possibly null) 2743 * (10) dvma 2744 * (11) devices 2745 */ 2746 static void 2747 kvm_init(void) 2748 { 2749 /* 2750 * Put the kernel segments in kernel address space. 2751 */ 2752 rw_enter(&kas.a_lock, RW_WRITER); 2753 as_avlinit(&kas); 2754 2755 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2756 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2757 (void) segkmem_create(&ktextseg); 2758 2759 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2760 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2761 (void) segkmem_create(&ktexthole); 2762 2763 (void) seg_attach(&kas, (caddr_t)valloc_base, 2764 (size_t)(econtig32 - valloc_base), &kvalloc); 2765 (void) segkmem_create(&kvalloc); 2766 2767 if (kmem64_base) { 2768 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2769 (size_t)(kmem64_end - kmem64_base), &kmem64); 2770 (void) segkmem_create(&kmem64); 2771 } 2772 2773 /* 2774 * We're about to map out /boot. This is the beginning of the 2775 * system resource management transition. We can no longer 2776 * call into /boot for I/O or memory allocations. 2777 */ 2778 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2779 (void) segkmem_create(&kvseg); 2780 hblk_alloc_dynamic = 1; 2781 2782 /* 2783 * we need to preallocate pages for DR operations before enabling large 2784 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2785 */ 2786 memseg_remap_init(); 2787 2788 /* at this point we are ready to use large page heap */ 2789 segkmem_heap_lp_init(); 2790 2791 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2792 &kvseg32); 2793 (void) segkmem_create(&kvseg32); 2794 2795 /* 2796 * Create a segment for the debugger. 2797 */ 2798 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2799 (void) segkmem_create(&kdebugseg); 2800 2801 rw_exit(&kas.a_lock); 2802 } 2803 2804 char obp_tte_str[] = 2805 "h# %x constant MMU_PAGESHIFT " 2806 "h# %x constant TTE8K " 2807 "h# %x constant SFHME_SIZE " 2808 "h# %x constant SFHME_TTE " 2809 "h# %x constant HMEBLK_TAG " 2810 "h# %x constant HMEBLK_NEXT " 2811 "h# %x constant HMEBLK_MISC " 2812 "h# %x constant HMEBLK_HME1 " 2813 "h# %x constant NHMENTS " 2814 "h# %x constant HBLK_SZMASK " 2815 "h# %x constant HBLK_RANGE_SHIFT " 2816 "h# %x constant HMEBP_HBLK " 2817 "h# %x constant HMEBLK_ENDPA " 2818 "h# %x constant HMEBUCKET_SIZE " 2819 "h# %x constant HTAG_SFMMUPSZ " 2820 "h# %x constant HTAG_BSPAGE_SHIFT " 2821 "h# %x constant HTAG_REHASH_SHIFT " 2822 "h# %x constant SFMMU_INVALID_SHMERID " 2823 "h# %x constant mmu_hashcnt " 2824 "h# %p constant uhme_hash " 2825 "h# %p constant khme_hash " 2826 "h# %x constant UHMEHASH_SZ " 2827 "h# %x constant KHMEHASH_SZ " 2828 "h# %p constant KCONTEXT " 2829 "h# %p constant KHATID " 2830 "h# %x constant ASI_MEM " 2831 2832 ": PHYS-X@ ( phys -- data ) " 2833 " ASI_MEM spacex@ " 2834 "; " 2835 2836 ": PHYS-W@ ( phys -- data ) " 2837 " ASI_MEM spacew@ " 2838 "; " 2839 2840 ": PHYS-L@ ( phys -- data ) " 2841 " ASI_MEM spaceL@ " 2842 "; " 2843 2844 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2845 " 3 * MMU_PAGESHIFT + " 2846 "; " 2847 2848 ": TTE_IS_VALID ( ttep -- flag ) " 2849 " PHYS-X@ 0< " 2850 "; " 2851 2852 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2853 " dup TTE8K = if " 2854 " drop HBLK_RANGE_SHIFT " 2855 " else " 2856 " TTE_PAGE_SHIFT " 2857 " then " 2858 "; " 2859 2860 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2861 " tuck >> swap MMU_PAGESHIFT - << " 2862 "; " 2863 2864 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2865 " >> over xor swap ( hash sfmmup ) " 2866 " KHATID <> if ( hash ) " 2867 " UHMEHASH_SZ and ( bucket ) " 2868 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2869 " else ( hash ) " 2870 " KHMEHASH_SZ and ( bucket ) " 2871 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2872 " then ( hmebp ) " 2873 "; " 2874 2875 ": HME_HASH_TABLE_SEARCH " 2876 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2877 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2878 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2879 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2880 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2881 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2882 " else " 2883 " hmeblk_next + phys-x@ false " 2884 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2885 " then " 2886 " else " 2887 " hmeblk_next + phys-x@ false " 2888 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2889 " then " 2890 " else " 2891 " drop 0 true " 2892 " then " 2893 " until r> drop " 2894 "; " 2895 2896 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2897 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2898 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2899 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2900 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2901 "; " 2902 2903 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2904 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2905 " TTE8K = if ( hmeblkp addr ) " 2906 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2907 " else ( hmeblkp addr ) " 2908 " drop 0 ( hmeblkp 0 ) " 2909 " then ( hmeblkp hme-index ) " 2910 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2911 " SFHME_TTE + ( ttep ) " 2912 "; " 2913 2914 ": unix-tte ( addr cnum -- false | tte-data true ) " 2915 " KCONTEXT = if ( addr ) " 2916 " KHATID ( addr khatid ) " 2917 " else ( addr ) " 2918 " drop false exit ( false ) " 2919 " then " 2920 " ( addr khatid ) " 2921 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2922 " 2dup swap i HME_HASH_SHIFT " 2923 "( addr sfmmup sfmmup addr hmeshift ) " 2924 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2925 " over i 4 pick " 2926 "( addr sfmmup hmebp sfmmup rehash addr ) " 2927 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2928 " HME_HASH_TABLE_SEARCH " 2929 "( addr sfmmup { null | hmeblkp } ) " 2930 " ?dup if ( addr sfmmup hmeblkp ) " 2931 " nip swap HBLK_TO_TTEP ( ttep ) " 2932 " dup TTE_IS_VALID if ( valid-ttep ) " 2933 " PHYS-X@ true ( tte-data true ) " 2934 " else ( invalid-tte ) " 2935 " drop false ( false ) " 2936 " then ( false | tte-data true ) " 2937 " unloop exit ( false | tte-data true ) " 2938 " then ( addr sfmmup ) " 2939 " loop ( addr sfmmup ) " 2940 " 2drop false ( false ) " 2941 "; " 2942 ; 2943 2944 void 2945 create_va_to_tte(void) 2946 { 2947 char *bp; 2948 extern int khmehash_num, uhmehash_num; 2949 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2950 2951 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2952 2953 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2954 2955 /* 2956 * Teach obp how to parse our sw ttes. 2957 */ 2958 (void) sprintf(bp, obp_tte_str, 2959 MMU_PAGESHIFT, 2960 TTE8K, 2961 sizeof (struct sf_hment), 2962 OFFSET(struct sf_hment, hme_tte), 2963 OFFSET(struct hme_blk, hblk_tag), 2964 OFFSET(struct hme_blk, hblk_nextpa), 2965 OFFSET(struct hme_blk, hblk_misc), 2966 OFFSET(struct hme_blk, hblk_hme), 2967 NHMENTS, 2968 HBLK_SZMASK, 2969 HBLK_RANGE_SHIFT, 2970 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2971 HMEBLK_ENDPA, 2972 sizeof (struct hmehash_bucket), 2973 HTAG_SFMMUPSZ, 2974 HTAG_BSPAGE_SHIFT, 2975 HTAG_REHASH_SHIFT, 2976 SFMMU_INVALID_SHMERID, 2977 mmu_hashcnt, 2978 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2979 (caddr_t)va_to_pa((caddr_t)khme_hash), 2980 UHMEHASH_SZ, 2981 KHMEHASH_SZ, 2982 KCONTEXT, 2983 KHATID, 2984 ASI_MEM); 2985 prom_interpret(bp, 0, 0, 0, 0, 0); 2986 2987 kobj_free(bp, MMU_PAGESIZE); 2988 } 2989 2990 void 2991 install_va_to_tte(void) 2992 { 2993 /* 2994 * advise prom that it can use unix-tte 2995 */ 2996 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 2997 } 2998 2999 /* 3000 * Here we add "device-type=console" for /os-io node, for currently 3001 * our kernel console output only supports displaying text and 3002 * performing cursor-positioning operations (through kernel framebuffer 3003 * driver) and it doesn't support other functionalities required for a 3004 * standard "display" device as specified in 1275 spec. The main missing 3005 * interface defined by the 1275 spec is "draw-logo". 3006 * also see the comments above prom_stdout_is_framebuffer(). 3007 */ 3008 static char *create_node = 3009 "\" /\" find-device " 3010 "new-device " 3011 "\" os-io\" device-name " 3012 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3013 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3014 " 2>r swap 2 2r> ['] $callback catch if " 3015 " 2drop 3drop 0 " 3016 " then " 3017 "; " 3018 ": read ( adr,len -- #read ) " 3019 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3020 " ( retN ... ret1 N ) " 3021 " ?dup if " 3022 " swap >r 1- 0 ?do drop loop r> " 3023 " else " 3024 " -2 " 3025 " then " 3026 "; " 3027 ": write ( adr,len -- #written ) " 3028 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3029 " ( retN ... ret1 N ) " 3030 " ?dup if " 3031 " swap >r 1- 0 ?do drop loop r> " 3032 " else " 3033 " 0 " 3034 " then " 3035 "; " 3036 ": poll-tty ( -- ) ; " 3037 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3038 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3039 ": cb-give/take ( $method -- ) " 3040 " 0 -rot ['] $callback catch ?dup if " 3041 " >r 2drop 2drop r> throw " 3042 " else " 3043 " 0 ?do drop loop " 3044 " then " 3045 "; " 3046 ": give ( -- ) \" exit-input\" cb-give/take ; " 3047 ": take ( -- ) \" enter-input\" cb-give/take ; " 3048 ": open ( -- ok? ) true ; " 3049 ": close ( -- ) ; " 3050 "finish-device " 3051 "device-end "; 3052 3053 /* 3054 * Create the OBP input/output node (FCode serial driver). 3055 * It is needed for both USB console keyboard and for 3056 * the kernel terminal emulator. It is too early to check for a 3057 * kernel console compatible framebuffer now, so we create this 3058 * so that we're ready if we need to enable kernel terminal emulation. 3059 * 3060 * When the USB software takes over the input device at the time 3061 * consconfig runs, OBP's stdin is redirected to this node. 3062 * Whenever the FORTH user interface is used after this switch, 3063 * the node will call back into the kernel for console input. 3064 * If a serial device such as ttya or a UART with a Type 5 keyboard 3065 * attached is used, OBP takes over the serial device when the system 3066 * goes to the debugger after the system is booted. This sharing 3067 * of the relatively simple serial device is difficult but possible. 3068 * Sharing the USB host controller is impossible due its complexity. 3069 * 3070 * Similarly to USB keyboard input redirection, after consconfig_dacf 3071 * configures a kernel console framebuffer as the standard output 3072 * device, OBP's stdout is switched to to vector through the 3073 * /os-io node into the kernel terminal emulator. 3074 */ 3075 static void 3076 startup_create_io_node(void) 3077 { 3078 prom_interpret(create_node, 0, 0, 0, 0, 0); 3079 } 3080 3081 3082 /* 3083 * Must be defined in platform dependent code. 3084 */ 3085 extern caddr_t modtext; 3086 extern size_t modtext_sz; 3087 extern caddr_t moddata; 3088 3089 #define HEAPTEXT_ARENA(addr) \ 3090 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3091 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3092 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3093 3094 #define HEAPTEXT_OVERSIZED(addr) \ 3095 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3096 3097 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3098 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3099 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3100 3101 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3102 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3103 kmutex_t texthole_lock; 3104 3105 char kern_bootargs[OBP_MAXPATHLEN]; 3106 char kern_bootfile[OBP_MAXPATHLEN]; 3107 3108 void 3109 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3110 { 3111 uintptr_t addr, limit; 3112 3113 addr = HEAPTEXT_BASE; 3114 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3115 3116 /* 3117 * Before we initialize the text_arena, we want to punch holes in the 3118 * underlying heaptext_arena. This guarantees that for any text 3119 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3120 */ 3121 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3122 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3123 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3124 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3125 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3126 } 3127 3128 /* 3129 * Allocate one page at the oversize to break up the text region 3130 * from the oversized region. 3131 */ 3132 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3133 (void *)limit, (void *)(limit + PAGESIZE), 3134 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3135 3136 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3137 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3138 heaptext_arena, 0, VM_SLEEP); 3139 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3140 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3141 } 3142 3143 caddr_t 3144 kobj_text_alloc(vmem_t *arena, size_t size) 3145 { 3146 caddr_t rval, better; 3147 3148 /* 3149 * First, try a sleeping allocation. 3150 */ 3151 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3152 3153 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3154 return (rval); 3155 3156 /* 3157 * We didn't get the area that we wanted. We're going to try to do an 3158 * allocation with explicit constraints. 3159 */ 3160 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3161 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3162 VM_NOSLEEP | VM_BESTFIT); 3163 3164 if (better != NULL) { 3165 /* 3166 * That worked. Free our first attempt and return. 3167 */ 3168 vmem_free(arena, rval, size); 3169 return (better); 3170 } 3171 3172 /* 3173 * That didn't work; we'll have to return our first attempt. 3174 */ 3175 return (rval); 3176 } 3177 3178 caddr_t 3179 kobj_texthole_alloc(caddr_t addr, size_t size) 3180 { 3181 int arena = HEAPTEXT_ARENA(addr); 3182 char c[30]; 3183 uintptr_t base; 3184 3185 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3186 /* 3187 * If this is an oversized allocation or it is allocated in 3188 * the nucleus data page, there is no text hole available for 3189 * it; return NULL. 3190 */ 3191 return (NULL); 3192 } 3193 3194 mutex_enter(&texthole_lock); 3195 3196 if (texthole_arena[arena] == NULL) { 3197 ASSERT(texthole_source[arena] == NULL); 3198 3199 if (arena == 0) { 3200 texthole_source[0] = vmem_create("module_text_holesrc", 3201 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3202 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3203 0, VM_SLEEP); 3204 } else { 3205 base = HEAPTEXT_BASE + 3206 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3207 3208 (void) snprintf(c, sizeof (c), 3209 "heaptext_holesrc_%d", arena); 3210 3211 texthole_source[arena] = vmem_create(c, (void *)base, 3212 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3213 0, VM_SLEEP); 3214 } 3215 3216 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3217 3218 texthole_arena[arena] = vmem_create(c, NULL, 0, 3219 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3220 texthole_source[arena], 0, VM_SLEEP); 3221 } 3222 3223 mutex_exit(&texthole_lock); 3224 3225 ASSERT(texthole_arena[arena] != NULL); 3226 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3227 return (vmem_alloc(texthole_arena[arena], size, 3228 VM_BESTFIT | VM_NOSLEEP)); 3229 } 3230 3231 void 3232 kobj_texthole_free(caddr_t addr, size_t size) 3233 { 3234 int arena = HEAPTEXT_ARENA(addr); 3235 3236 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3237 ASSERT(texthole_arena[arena] != NULL); 3238 vmem_free(texthole_arena[arena], addr, size); 3239 } 3240 3241 void 3242 release_bootstrap(void) 3243 { 3244 if (&cif_init) 3245 cif_init(); 3246 } 3247