1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* Copyright (c) 1988 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * Copyright 2019 Joyent, Inc. 31 * Copyright 2026 Oxide Computer Company 32 */ 33 34 #include <sys/types.h> 35 #include <sys/param.h> 36 #include <sys/sysmacros.h> 37 #include <sys/pcb.h> 38 #include <sys/systm.h> 39 #include <sys/signal.h> 40 #include <sys/cred.h> 41 #include <sys/user.h> 42 #include <sys/vfs.h> 43 #include <sys/vnode.h> 44 #include <sys/proc.h> 45 #include <sys/time.h> 46 #include <sys/file.h> 47 #include <sys/priocntl.h> 48 #include <sys/procset.h> 49 #include <sys/disp.h> 50 #include <sys/callo.h> 51 #include <sys/callb.h> 52 #include <sys/debug.h> 53 #include <sys/conf.h> 54 #include <sys/bootconf.h> 55 #include <sys/utsname.h> 56 #include <sys/cmn_err.h> 57 #include <sys/vmparam.h> 58 #include <sys/modctl.h> 59 #include <sys/vm.h> 60 #include <sys/callb.h> 61 #include <sys/ddi_periodic.h> 62 #include <sys/kmem.h> 63 #include <sys/vmem.h> 64 #include <sys/cpuvar.h> 65 #include <sys/cladm.h> 66 #include <sys/corectl.h> 67 #include <sys/exec.h> 68 #include <sys/syscall.h> 69 #include <sys/reboot.h> 70 #include <sys/task.h> 71 #include <sys/exacct.h> 72 #include <sys/autoconf.h> 73 #include <sys/errorq.h> 74 #include <sys/class.h> 75 #include <sys/stack.h> 76 #include <sys/brand.h> 77 #include <sys/mmapobj.h> 78 #include <sys/smt.h> 79 80 #include <vm/as.h> 81 #include <vm/seg_kmem.h> 82 #include <sys/dc_ki.h> 83 84 #include <c2/audit.h> 85 #include <sys/bootprops.h> 86 87 /* well known processes */ 88 proc_t *proc_sched; /* memory scheduler */ 89 proc_t *proc_init; /* init */ 90 proc_t *proc_pageout; /* pageout daemon */ 91 proc_t *proc_fsflush; /* fsflush daemon */ 92 93 pgcnt_t maxmem; /* Maximum available memory in pages. */ 94 pgcnt_t freemem; /* Current available memory in pages. */ 95 int interrupts_unleashed; /* set when we do the first spl0() */ 96 97 kmem_cache_t *process_cache; /* kmem cache for proc structures */ 98 99 /* 100 * Indicates whether the auditing module (c2audit) is loaded. Possible 101 * values are: 102 * 0 - c2audit module is excluded in /etc/system and cannot be loaded 103 * 1 - c2audit module is not loaded but can be anytime 104 * 2 - c2audit module is loaded 105 */ 106 int audit_active = C2AUDIT_DISABLED; 107 108 /* 109 * Process 0's lwp directory and lwpid hash table. 110 */ 111 lwpdir_t p0_lwpdir[2]; 112 tidhash_t p0_tidhash[2]; 113 lwpent_t p0_lep; 114 115 /* 116 * Machine-independent initialization code 117 * Called from cold start routine as 118 * soon as a stack and segmentation 119 * have been established. 120 * Functions: 121 * clear and free user core 122 * turn on clock 123 * hand craft 0th process 124 * call all initialization routines 125 * fork - process 0 to schedule 126 * - process 1 execute bootstrap 127 * - process 2 to page out 128 * create system threads 129 */ 130 131 int cluster_bootflags = 0; 132 133 void 134 cluster_wrapper(void) 135 { 136 cluster(); 137 panic("cluster() returned"); 138 } 139 140 char initname[INITNAME_SZ] = "/sbin/init"; /* also referenced by zone0 */ 141 char initargs[BOOTARGS_MAX] = ""; /* also referenced by zone0 */ 142 143 /* 144 * Construct a stack for init containing the arguments to it, then 145 * pass control to exec_common. 146 */ 147 int 148 exec_init(const char *initpath, const char *args) 149 { 150 uintptr_t ucp; 151 uintptr_t uap; 152 uintptr_t *argv; 153 uintptr_t exec_fnamep; 154 char *scratchargs; 155 int i, sarg; 156 size_t argvlen, alen; 157 size_t wlen = sizeof (uintptr_t); 158 boolean_t in_arg; 159 int argc = 0; 160 int error = 0, count = 0; 161 proc_t *p = ttoproc(curthread); 162 klwp_t *lwp = ttolwp(curthread); 163 int brand_action; 164 165 if (args == NULL) 166 args = ""; 167 168 alen = strlen(initpath) + 1 + strlen(args) + 1; 169 scratchargs = kmem_alloc(alen, KM_SLEEP); 170 (void) snprintf(scratchargs, alen, "%s %s", initpath, args); 171 172 /* 173 * We do a quick two state parse of the string to sort out how big 174 * argc should be. 175 */ 176 in_arg = B_FALSE; 177 for (i = 0; i < strlen(scratchargs); i++) { 178 if (scratchargs[i] == ' ' || scratchargs[i] == '\0') { 179 if (in_arg) { 180 in_arg = B_FALSE; 181 argc++; 182 } 183 } else { 184 in_arg = B_TRUE; 185 } 186 } 187 argvlen = sizeof (uintptr_t) * (argc + 1); 188 argv = kmem_zalloc(argvlen, KM_SLEEP); 189 190 /* 191 * We pull off a bit of a hack here. We work our way through the 192 * args string, putting nulls at the ends of space delimited tokens 193 * (boot args don't support quoting at this time). Then we just 194 * copy the whole mess to userland in one go. In other words, we 195 * transform this: "init -s -r\0" into this on the stack: 196 * 197 * -0x00 \0 198 * -0x01 r 199 * -0x02 - <--------. 200 * -0x03 \0 | 201 * -0x04 s | 202 * -0x05 - <------. | 203 * -0x06 \0 | | 204 * -0x07 t | | 205 * -0x08 i | | 206 * -0x09 n | | 207 * -0x0a i <---. | | 208 * -0x10 NULL | | | (argv[3]) 209 * -0x14 -----|--|-' (argv[2]) 210 * -0x18 ------|--' (argv[1]) 211 * -0x1c -------' (argv[0]) 212 * 213 * Since we know the value of ucp at the beginning of this process, 214 * we can trivially compute the argv[] array which we also need to 215 * place in userland: argv[i] = ucp - sarg(i), where ucp is the 216 * stack ptr, and sarg is the string index of the start of the 217 * argument. 218 */ 219 ucp = (uintptr_t)p->p_usrstack; 220 221 argc = 0; 222 in_arg = B_FALSE; 223 sarg = 0; 224 225 for (i = 0; i < alen; i++) { 226 if (scratchargs[i] == ' ' || scratchargs[i] == '\0') { 227 if (in_arg == B_TRUE) { 228 in_arg = B_FALSE; 229 scratchargs[i] = '\0'; 230 argv[argc++] = ucp - (alen - sarg); 231 } 232 } else if (in_arg == B_FALSE) { 233 in_arg = B_TRUE; 234 sarg = i; 235 } 236 } 237 238 exec_fnamep = argv[0]; 239 240 ucp -= alen; 241 error |= copyout(scratchargs, (caddr_t)ucp, alen); 242 243 if (p->p_model == DATAMODEL_ILP32) { 244 uintptr32_t *argv32; 245 246 argv32 = kmem_zalloc(argvlen / 2, KM_SLEEP); 247 248 for (i = 0; i < argc; i++) 249 argv32[i] = (uintptr32_t)argv[i]; 250 251 kmem_free(argv, argvlen); 252 argv = (uintptr_t *)argv32; 253 argvlen /= 2; 254 255 wlen = sizeof (uintptr32_t); 256 } 257 258 uap = P2ALIGN(ucp, wlen); 259 /* advance to be below the word we're in */ 260 uap -= wlen; 261 /* advance argc words down, plus one for NULL */ 262 uap -= (argc + 1) * wlen; 263 error |= copyout(argv, (caddr_t)uap, argvlen); 264 265 if (error != 0) { 266 zcmn_err(p->p_zone->zone_id, CE_WARN, 267 "Could not construct stack for init.\n"); 268 kmem_free(argv, argvlen); 269 kmem_free(scratchargs, alen); 270 return (EFAULT); 271 } 272 273 kmem_free(argv, argvlen); 274 kmem_free(scratchargs, alen); 275 276 /* 277 * Point at the arguments. 278 */ 279 lwp->lwp_ap = lwp->lwp_arg; 280 lwp->lwp_arg[0] = exec_fnamep; 281 lwp->lwp_arg[1] = uap; 282 lwp->lwp_arg[2] = 0; 283 curthread->t_post_sys = 1; 284 curthread->t_sysnum = SYS_execve; 285 286 /* 287 * If we are executing init from zsched, we may have inherited its 288 * parent process's signal mask. Clear it now so that we behave in 289 * the same way as when started from the global zone. 290 */ 291 sigemptyset(&curthread->t_hold); 292 293 brand_action = ZONE_IS_BRANDED(p->p_zone) ? EBA_BRAND : EBA_NONE; 294 again: 295 error = exec_common((const char *)exec_fnamep, 296 (const char **)uap, NULL, NULL, brand_action, UIO_USERSPACE); 297 298 /* 299 * Normally we would just set lwp_argsaved and t_post_sys and 300 * let post_syscall reset lwp_ap for us. Unfortunately, 301 * exec_init isn't always called from a system call. Instead 302 * of making a mess of trap_cleanup, we just reset the args 303 * pointer here. 304 */ 305 reset_syscall_args(); 306 307 switch (error) { 308 case 0: 309 return (0); 310 311 case ENOENT: 312 zcmn_err(p->p_zone->zone_id, CE_WARN, 313 "exec(%s) failed (file not found).\n", initpath); 314 return (ENOENT); 315 316 case EAGAIN: 317 case EINTR: 318 ++count; 319 if (count < 5) { 320 zcmn_err(p->p_zone->zone_id, CE_WARN, 321 "exec(%s) failed with errno %d. Retrying...\n", 322 initpath, error); 323 goto again; 324 } 325 } 326 327 zcmn_err(p->p_zone->zone_id, CE_WARN, 328 "exec(%s) failed with errno %d.", initpath, error); 329 return (error); 330 } 331 332 /* 333 * This routine does all of the common setup for invoking init; global 334 * and non-global zones employ this routine for the functionality which is 335 * in common. 336 * 337 * This program (init, presumably) must be a 32-bit process. 338 */ 339 int 340 start_init_common() 341 { 342 proc_t *p = curproc; 343 ASSERT_STACK_ALIGNED(); 344 p->p_zone->zone_proc_initpid = p->p_pid; 345 346 p->p_cstime = p->p_stime = p->p_cutime = p->p_utime = 0; 347 p->p_usrstack = (caddr_t)USRSTACK32; 348 p->p_model = DATAMODEL_ILP32; 349 p->p_stkprot = PROT_ZFOD & ~PROT_EXEC; 350 p->p_datprot = PROT_ZFOD & ~PROT_EXEC; 351 p->p_stk_ctl = INT32_MAX; 352 353 p->p_as = as_alloc(); 354 p->p_as->a_proc = p; 355 p->p_as->a_userlimit = (caddr_t)USERLIMIT32; 356 (void) hat_setup(p->p_as->a_hat, HAT_INIT); 357 358 init_core(); 359 360 init_mstate(curthread, LMS_SYSTEM); 361 return (exec_init(p->p_zone->zone_initname, p->p_zone->zone_bootargs)); 362 } 363 364 /* 365 * Start the initial user process for the global zone; once running, if 366 * init should subsequently fail, it will be automatically be caught in the 367 * exit(2) path, and restarted by restart_init(). 368 */ 369 static void 370 start_init(void) 371 { 372 proc_init = curproc; 373 374 ASSERT(curproc->p_zone->zone_initname != NULL); 375 376 if (start_init_common() != 0) 377 halt("unix: Could not start init"); 378 lwp_rtt(); 379 } 380 381 void 382 main(void) 383 { 384 proc_t *p = ttoproc(curthread); /* &p0 */ 385 int (**initptr)(); 386 extern void sched(); 387 extern void fsflush(); 388 extern int (*init_tbl[])(); 389 extern int (*mp_init_tbl[])(); 390 extern id_t syscid, defaultcid; 391 extern int swaploaded; 392 extern int netboot; 393 extern ib_boot_prop_t *iscsiboot_prop; 394 extern void vm_init(void); 395 extern void cbe_init_pre(void); 396 extern void cbe_init(void); 397 extern void clock_tick_init_pre(void); 398 extern void clock_tick_init_post(void); 399 extern void clock_init(void); 400 extern void physio_bufs_init(void); 401 extern void pm_cfb_setup_intr(void); 402 extern int pm_adjust_timestamps(dev_info_t *, void *); 403 extern void start_other_cpus(int); 404 extern void sysevent_evc_thrinit(); 405 extern kmutex_t ualock; 406 #if defined(__x86) 407 extern void fastboot_post_startup(void); 408 extern void progressbar_start(void); 409 #endif 410 /* 411 * In the horrible world of x86 in-lines, you can't get symbolic 412 * structure offsets a la genassym. This assertion is here so 413 * that the next poor slob who innocently changes the offset of 414 * cpu_thread doesn't waste as much time as I just did finding 415 * out that it's hard-coded in i86/ml/i86.il. Similarly for 416 * curcpup. You're welcome. 417 */ 418 ASSERT(CPU == CPU->cpu_self); 419 ASSERT(curthread == CPU->cpu_thread); 420 ASSERT_STACK_ALIGNED(); 421 422 /* 423 * We take the ualock until we have completed the startup 424 * to prevent kadmin() from disrupting this work. In particular, 425 * we don't want kadmin() to bring the system down while we are 426 * trying to start it up. 427 */ 428 mutex_enter(&ualock); 429 430 /* 431 * Setup root lgroup and leaf lgroup for CPU 0 432 */ 433 lgrp_init(LGRP_INIT_STAGE2); 434 435 /* 436 * Once 'startup()' completes, the thread_reaper() daemon would be 437 * created(in thread_init()). After that, it is safe to create threads 438 * that could exit. These exited threads will get reaped. 439 */ 440 startup(); 441 segkmem_gc(); 442 callb_init(); 443 cbe_init_pre(); /* x86 must initialize gethrtimef before timer_init */ 444 ddi_periodic_init(); 445 cbe_init(); 446 callout_init(); /* callout table MUST be init'd after cyclics */ 447 clock_tick_init_pre(); 448 clock_init(); 449 450 #if defined(__x86) 451 /* 452 * The progressbar thread uses cv_reltimedwait() and hence needs to be 453 * started after the callout mechanism has been initialized. 454 */ 455 progressbar_start(); 456 #endif 457 /* 458 * On some platforms, clkinitf() changes the timing source that 459 * gethrtime_unscaled() uses to generate timestamps. cbe_init() calls 460 * clkinitf(), so re-initialize the microstate counters after the 461 * timesource has been chosen. 462 */ 463 init_mstate(&t0, LMS_SYSTEM); 464 init_cpu_mstate(CPU, CMS_SYSTEM); 465 466 /* 467 * May need to probe to determine latencies from CPU 0 after 468 * gethrtime() comes alive in cbe_init() and before enabling interrupts 469 * and copy and release any temporary memory allocated with BOP_ALLOC() 470 * before release_bootstrap() frees boot memory 471 */ 472 lgrp_init(LGRP_INIT_STAGE3); 473 474 /* 475 * Call all system initialization functions. 476 */ 477 for (initptr = &init_tbl[0]; *initptr; initptr++) 478 (**initptr)(); 479 /* 480 * Load iSCSI boot properties 481 */ 482 ld_ib_prop(); 483 /* 484 * initialize vm related stuff. 485 */ 486 vm_init(); 487 488 /* 489 * initialize buffer pool for raw I/O requests 490 */ 491 physio_bufs_init(); 492 493 ttolwp(curthread)->lwp_error = 0; /* XXX kludge for SCSI driver */ 494 495 /* 496 * Drop the interrupt level and allow interrupts. At this point 497 * the DDI guarantees that interrupts are enabled. 498 */ 499 (void) spl0(); 500 interrupts_unleashed = 1; 501 502 /* 503 * Create kmem cache for proc structures 504 */ 505 process_cache = kmem_cache_create("process_cache", sizeof (proc_t), 506 0, NULL, NULL, NULL, NULL, NULL, 0); 507 508 vfs_mountroot(); /* Mount the root file system */ 509 errorq_init(); /* after vfs_mountroot() so DDI root is ready */ 510 cpu_kstat_init(CPU); /* after vfs_mountroot() so TOD is valid */ 511 ddi_walk_devs(ddi_root_node(), pm_adjust_timestamps, NULL); 512 /* after vfs_mountroot() so hrestime is valid */ 513 514 post_startup(); 515 swaploaded = 1; 516 517 /* 518 * Initialize Solaris Audit Subsystem 519 */ 520 audit_init(); 521 522 /* 523 * Start the periodic hash rescale for all vmem arenas before we load 524 * protocol modules and drivers via strplumb() below. Some drivers 525 * might rely on heavy vmem operations that could hurt performance 526 * without the rescale. 527 */ 528 vmem_update(NULL); 529 530 /* 531 * Plumb the protocol modules and drivers only if we are not 532 * networked booted, in this case we already did it in rootconf(). 533 */ 534 if (netboot == 0 && iscsiboot_prop == NULL) 535 (void) strplumb(); 536 537 gethrestime(&PTOU(curproc)->u_start); 538 curthread->t_start = PTOU(curproc)->u_start.tv_sec; 539 p->p_mstart = gethrtime(); 540 541 /* 542 * Perform setup functions that can only be done after root 543 * and swap have been set up. 544 */ 545 consconfig(); 546 #ifndef __sparc 547 release_bootstrap(); 548 #endif 549 550 /* 551 * attach drivers with ddi-forceattach prop 552 * It must be done early enough to load hotplug drivers (e.g. 553 * pcmcia nexus) so that devices enumerated via hotplug is 554 * available before I/O subsystem is fully initialized. 555 */ 556 i_ddi_forceattach_drivers(); 557 558 /* 559 * Set the scan rate and other parameters of the paging subsystem. 560 */ 561 setupclock(); 562 563 /* 564 * Initialize process 0's lwp directory and lwpid hash table. 565 */ 566 p->p_lwpdir = p->p_lwpfree = p0_lwpdir; 567 p->p_lwpdir->ld_next = p->p_lwpdir + 1; 568 p->p_lwpdir_sz = 2; 569 p->p_tidhash = p0_tidhash; 570 p->p_tidhash_sz = 2; 571 p0_lep.le_thread = curthread; 572 p0_lep.le_lwpid = curthread->t_tid; 573 p0_lep.le_start = curthread->t_start; 574 lwp_hash_in(p, &p0_lep, p0_tidhash, 2, 0); 575 576 /* 577 * Initialize extended accounting. 578 */ 579 exacct_init(); 580 581 /* 582 * Initialize threads of sysevent event channels 583 */ 584 sysevent_evc_thrinit(); 585 586 /* 587 * This must be done after post_startup() but before 588 * start_other_cpus() 589 */ 590 lgrp_init(LGRP_INIT_STAGE4); 591 592 /* 593 * Perform MP initialization, if any. 594 */ 595 start_other_cpus(0); 596 597 #ifdef __sparc 598 /* 599 * Release bootstrap here since PROM interfaces are 600 * used to start other CPUs above. 601 */ 602 release_bootstrap(); 603 #endif 604 605 /* 606 * Finish lgrp initialization after all CPUS are brought online. 607 */ 608 lgrp_init(LGRP_INIT_STAGE5); 609 610 /* 611 * After mp_init(), number of cpus are known (this is 612 * true for the time being, when there are actually 613 * hot pluggable cpus then this scheme would not do). 614 * Any per cpu initialization is done here. 615 */ 616 kmem_mp_init(); 617 618 clock_tick_init_post(); 619 620 for (initptr = &mp_init_tbl[0]; *initptr; initptr++) 621 (**initptr)(); 622 623 /* 624 * These must be called after start_other_cpus 625 */ 626 pm_cfb_setup_intr(); 627 #if defined(__x86) 628 fastboot_post_startup(); 629 630 smt_late_init(); 631 #endif 632 633 /* 634 * Make init process; enter scheduling loop with system process. 635 * 636 * Note that we manually assign the pids for these processes, for 637 * historical reasons. If more pre-assigned pids are needed, 638 * FAMOUS_PIDS will have to be updated. 639 */ 640 641 /* create init process */ 642 if (newproc(start_init, NULL, defaultcid, 59, NULL, 643 FAMOUS_PID_INIT)) 644 panic("main: unable to fork init."); 645 646 /* create pageout daemon */ 647 if (newproc(pageout, NULL, syscid, maxclsyspri - 1, NULL, 648 FAMOUS_PID_PAGEOUT)) 649 panic("main: unable to fork pageout()"); 650 651 /* create fsflush daemon */ 652 if (newproc(fsflush, NULL, syscid, minclsyspri, NULL, 653 FAMOUS_PID_FSFLUSH)) 654 panic("main: unable to fork fsflush()"); 655 656 /* create cluster process if we're a member of one */ 657 if (cluster_bootflags & CLUSTER_BOOTED) { 658 if (newproc(cluster_wrapper, NULL, syscid, minclsyspri, 659 NULL, 0)) { 660 panic("main: unable to fork cluster()"); 661 } 662 } 663 664 /* 665 * Create system threads (threads are associated with p0) 666 */ 667 668 /* create module uninstall daemon */ 669 /* BugID 1132273. If swapping over NFS need a bigger stack */ 670 (void) thread_create(NULL, 0, (void (*)())mod_uninstall_daemon, 671 NULL, 0, &p0, TS_RUN, minclsyspri); 672 673 (void) thread_create(NULL, 0, seg_pasync_thread, 674 NULL, 0, &p0, TS_RUN, minclsyspri); 675 676 pid_setmin(); 677 678 /* system is now ready */ 679 mutex_exit(&ualock); 680 681 bcopy("sched", PTOU(curproc)->u_psargs, 6); 682 bcopy("sched", PTOU(curproc)->u_comm, 5); 683 sched(); 684 /* NOTREACHED */ 685 } 686