1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /* Copyright (c) 1988 AT&T */
27 /* All Rights Reserved */
28
29 /*
30 * Copyright 2019 Joyent, Inc.
31 * Copyright 2026 Oxide Computer Company
32 */
33
34 #include <sys/types.h>
35 #include <sys/param.h>
36 #include <sys/sysmacros.h>
37 #include <sys/pcb.h>
38 #include <sys/systm.h>
39 #include <sys/signal.h>
40 #include <sys/cred.h>
41 #include <sys/user.h>
42 #include <sys/vfs.h>
43 #include <sys/vnode.h>
44 #include <sys/proc.h>
45 #include <sys/time.h>
46 #include <sys/file.h>
47 #include <sys/priocntl.h>
48 #include <sys/procset.h>
49 #include <sys/disp.h>
50 #include <sys/callo.h>
51 #include <sys/callb.h>
52 #include <sys/debug.h>
53 #include <sys/conf.h>
54 #include <sys/bootconf.h>
55 #include <sys/utsname.h>
56 #include <sys/cmn_err.h>
57 #include <sys/vmparam.h>
58 #include <sys/modctl.h>
59 #include <sys/vm.h>
60 #include <sys/callb.h>
61 #include <sys/ddi_periodic.h>
62 #include <sys/kmem.h>
63 #include <sys/vmem.h>
64 #include <sys/cpuvar.h>
65 #include <sys/cladm.h>
66 #include <sys/corectl.h>
67 #include <sys/exec.h>
68 #include <sys/syscall.h>
69 #include <sys/reboot.h>
70 #include <sys/task.h>
71 #include <sys/exacct.h>
72 #include <sys/autoconf.h>
73 #include <sys/errorq.h>
74 #include <sys/class.h>
75 #include <sys/stack.h>
76 #include <sys/brand.h>
77 #include <sys/mmapobj.h>
78 #include <sys/smt.h>
79
80 #include <vm/as.h>
81 #include <vm/seg_kmem.h>
82 #include <sys/dc_ki.h>
83
84 #include <c2/audit.h>
85 #include <sys/bootprops.h>
86
87 /* well known processes */
88 proc_t *proc_sched; /* memory scheduler */
89 proc_t *proc_init; /* init */
90 proc_t *proc_pageout; /* pageout daemon */
91 proc_t *proc_fsflush; /* fsflush daemon */
92
93 pgcnt_t maxmem; /* Maximum available memory in pages. */
94 pgcnt_t freemem; /* Current available memory in pages. */
95 int interrupts_unleashed; /* set when we do the first spl0() */
96
97 kmem_cache_t *process_cache; /* kmem cache for proc structures */
98
99 /*
100 * Indicates whether the auditing module (c2audit) is loaded. Possible
101 * values are:
102 * 0 - c2audit module is excluded in /etc/system and cannot be loaded
103 * 1 - c2audit module is not loaded but can be anytime
104 * 2 - c2audit module is loaded
105 */
106 int audit_active = C2AUDIT_DISABLED;
107
108 /*
109 * Process 0's lwp directory and lwpid hash table.
110 */
111 lwpdir_t p0_lwpdir[2];
112 tidhash_t p0_tidhash[2];
113 lwpent_t p0_lep;
114
115 /*
116 * Machine-independent initialization code
117 * Called from cold start routine as
118 * soon as a stack and segmentation
119 * have been established.
120 * Functions:
121 * clear and free user core
122 * turn on clock
123 * hand craft 0th process
124 * call all initialization routines
125 * fork - process 0 to schedule
126 * - process 1 execute bootstrap
127 * - process 2 to page out
128 * create system threads
129 */
130
131 int cluster_bootflags = 0;
132
133 void
cluster_wrapper(void)134 cluster_wrapper(void)
135 {
136 cluster();
137 panic("cluster() returned");
138 }
139
140 char initname[INITNAME_SZ] = "/sbin/init"; /* also referenced by zone0 */
141 char initargs[BOOTARGS_MAX] = ""; /* also referenced by zone0 */
142
143 /*
144 * Construct a stack for init containing the arguments to it, then
145 * pass control to exec_common.
146 */
147 int
exec_init(const char * initpath,const char * args)148 exec_init(const char *initpath, const char *args)
149 {
150 uintptr_t ucp;
151 uintptr_t uap;
152 uintptr_t *argv;
153 uintptr_t exec_fnamep;
154 char *scratchargs;
155 int i, sarg;
156 size_t argvlen, alen;
157 size_t wlen = sizeof (uintptr_t);
158 boolean_t in_arg;
159 int argc = 0;
160 int error = 0, count = 0;
161 proc_t *p = ttoproc(curthread);
162 klwp_t *lwp = ttolwp(curthread);
163 int brand_action;
164
165 if (args == NULL)
166 args = "";
167
168 alen = strlen(initpath) + 1 + strlen(args) + 1;
169 scratchargs = kmem_alloc(alen, KM_SLEEP);
170 (void) snprintf(scratchargs, alen, "%s %s", initpath, args);
171
172 /*
173 * We do a quick two state parse of the string to sort out how big
174 * argc should be.
175 */
176 in_arg = B_FALSE;
177 for (i = 0; i < strlen(scratchargs); i++) {
178 if (scratchargs[i] == ' ' || scratchargs[i] == '\0') {
179 if (in_arg) {
180 in_arg = B_FALSE;
181 argc++;
182 }
183 } else {
184 in_arg = B_TRUE;
185 }
186 }
187 argvlen = sizeof (uintptr_t) * (argc + 1);
188 argv = kmem_zalloc(argvlen, KM_SLEEP);
189
190 /*
191 * We pull off a bit of a hack here. We work our way through the
192 * args string, putting nulls at the ends of space delimited tokens
193 * (boot args don't support quoting at this time). Then we just
194 * copy the whole mess to userland in one go. In other words, we
195 * transform this: "init -s -r\0" into this on the stack:
196 *
197 * -0x00 \0
198 * -0x01 r
199 * -0x02 - <--------.
200 * -0x03 \0 |
201 * -0x04 s |
202 * -0x05 - <------. |
203 * -0x06 \0 | |
204 * -0x07 t | |
205 * -0x08 i | |
206 * -0x09 n | |
207 * -0x0a i <---. | |
208 * -0x10 NULL | | | (argv[3])
209 * -0x14 -----|--|-' (argv[2])
210 * -0x18 ------|--' (argv[1])
211 * -0x1c -------' (argv[0])
212 *
213 * Since we know the value of ucp at the beginning of this process,
214 * we can trivially compute the argv[] array which we also need to
215 * place in userland: argv[i] = ucp - sarg(i), where ucp is the
216 * stack ptr, and sarg is the string index of the start of the
217 * argument.
218 */
219 ucp = (uintptr_t)p->p_usrstack;
220
221 argc = 0;
222 in_arg = B_FALSE;
223 sarg = 0;
224
225 for (i = 0; i < alen; i++) {
226 if (scratchargs[i] == ' ' || scratchargs[i] == '\0') {
227 if (in_arg == B_TRUE) {
228 in_arg = B_FALSE;
229 scratchargs[i] = '\0';
230 argv[argc++] = ucp - (alen - sarg);
231 }
232 } else if (in_arg == B_FALSE) {
233 in_arg = B_TRUE;
234 sarg = i;
235 }
236 }
237
238 exec_fnamep = argv[0];
239
240 ucp -= alen;
241 error |= copyout(scratchargs, (caddr_t)ucp, alen);
242
243 if (p->p_model == DATAMODEL_ILP32) {
244 uintptr32_t *argv32;
245
246 argv32 = kmem_zalloc(argvlen / 2, KM_SLEEP);
247
248 for (i = 0; i < argc; i++)
249 argv32[i] = (uintptr32_t)argv[i];
250
251 kmem_free(argv, argvlen);
252 argv = (uintptr_t *)argv32;
253 argvlen /= 2;
254
255 wlen = sizeof (uintptr32_t);
256 }
257
258 uap = P2ALIGN(ucp, wlen);
259 /* advance to be below the word we're in */
260 uap -= wlen;
261 /* advance argc words down, plus one for NULL */
262 uap -= (argc + 1) * wlen;
263 error |= copyout(argv, (caddr_t)uap, argvlen);
264
265 if (error != 0) {
266 zcmn_err(p->p_zone->zone_id, CE_WARN,
267 "Could not construct stack for init.\n");
268 kmem_free(argv, argvlen);
269 kmem_free(scratchargs, alen);
270 return (EFAULT);
271 }
272
273 kmem_free(argv, argvlen);
274 kmem_free(scratchargs, alen);
275
276 /*
277 * Point at the arguments.
278 */
279 lwp->lwp_ap = lwp->lwp_arg;
280 lwp->lwp_arg[0] = exec_fnamep;
281 lwp->lwp_arg[1] = uap;
282 lwp->lwp_arg[2] = 0;
283 curthread->t_post_sys = 1;
284 curthread->t_sysnum = SYS_execve;
285
286 /*
287 * If we are executing init from zsched, we may have inherited its
288 * parent process's signal mask. Clear it now so that we behave in
289 * the same way as when started from the global zone.
290 */
291 sigemptyset(&curthread->t_hold);
292
293 brand_action = ZONE_IS_BRANDED(p->p_zone) ? EBA_BRAND : EBA_NONE;
294 again:
295 error = exec_common((const char *)exec_fnamep,
296 (const char **)uap, NULL, NULL, brand_action, UIO_USERSPACE);
297
298 /*
299 * Normally we would just set lwp_argsaved and t_post_sys and
300 * let post_syscall reset lwp_ap for us. Unfortunately,
301 * exec_init isn't always called from a system call. Instead
302 * of making a mess of trap_cleanup, we just reset the args
303 * pointer here.
304 */
305 reset_syscall_args();
306
307 switch (error) {
308 case 0:
309 return (0);
310
311 case ENOENT:
312 zcmn_err(p->p_zone->zone_id, CE_WARN,
313 "exec(%s) failed (file not found).\n", initpath);
314 return (ENOENT);
315
316 case EAGAIN:
317 case EINTR:
318 ++count;
319 if (count < 5) {
320 zcmn_err(p->p_zone->zone_id, CE_WARN,
321 "exec(%s) failed with errno %d. Retrying...\n",
322 initpath, error);
323 goto again;
324 }
325 }
326
327 zcmn_err(p->p_zone->zone_id, CE_WARN,
328 "exec(%s) failed with errno %d.", initpath, error);
329 return (error);
330 }
331
332 /*
333 * This routine does all of the common setup for invoking init; global
334 * and non-global zones employ this routine for the functionality which is
335 * in common.
336 *
337 * This program (init, presumably) must be a 32-bit process.
338 */
339 int
start_init_common()340 start_init_common()
341 {
342 proc_t *p = curproc;
343 ASSERT_STACK_ALIGNED();
344 p->p_zone->zone_proc_initpid = p->p_pid;
345
346 p->p_cstime = p->p_stime = p->p_cutime = p->p_utime = 0;
347 p->p_usrstack = (caddr_t)USRSTACK32;
348 p->p_model = DATAMODEL_ILP32;
349 p->p_stkprot = PROT_ZFOD & ~PROT_EXEC;
350 p->p_datprot = PROT_ZFOD & ~PROT_EXEC;
351 p->p_stk_ctl = INT32_MAX;
352
353 p->p_as = as_alloc();
354 p->p_as->a_proc = p;
355 p->p_as->a_userlimit = (caddr_t)USERLIMIT32;
356 (void) hat_setup(p->p_as->a_hat, HAT_INIT);
357
358 init_core();
359
360 init_mstate(curthread, LMS_SYSTEM);
361 return (exec_init(p->p_zone->zone_initname, p->p_zone->zone_bootargs));
362 }
363
364 /*
365 * Start the initial user process for the global zone; once running, if
366 * init should subsequently fail, it will be automatically be caught in the
367 * exit(2) path, and restarted by restart_init().
368 */
369 static void
start_init(void)370 start_init(void)
371 {
372 proc_init = curproc;
373
374 ASSERT(curproc->p_zone->zone_initname != NULL);
375
376 if (start_init_common() != 0)
377 halt("unix: Could not start init");
378 lwp_rtt();
379 }
380
381 void
main(void)382 main(void)
383 {
384 proc_t *p = ttoproc(curthread); /* &p0 */
385 int (**initptr)();
386 extern void sched();
387 extern void fsflush();
388 extern int (*init_tbl[])();
389 extern int (*mp_init_tbl[])();
390 extern id_t syscid, defaultcid;
391 extern int swaploaded;
392 extern int netboot;
393 extern ib_boot_prop_t *iscsiboot_prop;
394 extern void vm_init(void);
395 extern void cbe_init_pre(void);
396 extern void cbe_init(void);
397 extern void clock_tick_init_pre(void);
398 extern void clock_tick_init_post(void);
399 extern void clock_init(void);
400 extern void physio_bufs_init(void);
401 extern void pm_cfb_setup_intr(void);
402 extern int pm_adjust_timestamps(dev_info_t *, void *);
403 extern void start_other_cpus(int);
404 extern void sysevent_evc_thrinit();
405 extern kmutex_t ualock;
406 #if defined(__x86)
407 extern void fastboot_post_startup(void);
408 extern void progressbar_start(void);
409 #endif
410 /*
411 * In the horrible world of x86 in-lines, you can't get symbolic
412 * structure offsets a la genassym. This assertion is here so
413 * that the next poor slob who innocently changes the offset of
414 * cpu_thread doesn't waste as much time as I just did finding
415 * out that it's hard-coded in i86/ml/i86.il. Similarly for
416 * curcpup. You're welcome.
417 */
418 ASSERT(CPU == CPU->cpu_self);
419 ASSERT(curthread == CPU->cpu_thread);
420 ASSERT_STACK_ALIGNED();
421
422 /*
423 * We take the ualock until we have completed the startup
424 * to prevent kadmin() from disrupting this work. In particular,
425 * we don't want kadmin() to bring the system down while we are
426 * trying to start it up.
427 */
428 mutex_enter(&ualock);
429
430 /*
431 * Setup root lgroup and leaf lgroup for CPU 0
432 */
433 lgrp_init(LGRP_INIT_STAGE2);
434
435 /*
436 * Once 'startup()' completes, the thread_reaper() daemon would be
437 * created(in thread_init()). After that, it is safe to create threads
438 * that could exit. These exited threads will get reaped.
439 */
440 startup();
441 segkmem_gc();
442 callb_init();
443 cbe_init_pre(); /* x86 must initialize gethrtimef before timer_init */
444 ddi_periodic_init();
445 cbe_init();
446 callout_init(); /* callout table MUST be init'd after cyclics */
447 clock_tick_init_pre();
448 clock_init();
449
450 #if defined(__x86)
451 /*
452 * The progressbar thread uses cv_reltimedwait() and hence needs to be
453 * started after the callout mechanism has been initialized.
454 */
455 progressbar_start();
456 #endif
457 /*
458 * On some platforms, clkinitf() changes the timing source that
459 * gethrtime_unscaled() uses to generate timestamps. cbe_init() calls
460 * clkinitf(), so re-initialize the microstate counters after the
461 * timesource has been chosen.
462 */
463 init_mstate(&t0, LMS_SYSTEM);
464 init_cpu_mstate(CPU, CMS_SYSTEM);
465
466 /*
467 * May need to probe to determine latencies from CPU 0 after
468 * gethrtime() comes alive in cbe_init() and before enabling interrupts
469 * and copy and release any temporary memory allocated with BOP_ALLOC()
470 * before release_bootstrap() frees boot memory
471 */
472 lgrp_init(LGRP_INIT_STAGE3);
473
474 /*
475 * Call all system initialization functions.
476 */
477 for (initptr = &init_tbl[0]; *initptr; initptr++)
478 (**initptr)();
479 /*
480 * Load iSCSI boot properties
481 */
482 ld_ib_prop();
483 /*
484 * initialize vm related stuff.
485 */
486 vm_init();
487
488 /*
489 * initialize buffer pool for raw I/O requests
490 */
491 physio_bufs_init();
492
493 ttolwp(curthread)->lwp_error = 0; /* XXX kludge for SCSI driver */
494
495 /*
496 * Drop the interrupt level and allow interrupts. At this point
497 * the DDI guarantees that interrupts are enabled.
498 */
499 (void) spl0();
500 interrupts_unleashed = 1;
501
502 /*
503 * Create kmem cache for proc structures
504 */
505 process_cache = kmem_cache_create("process_cache", sizeof (proc_t),
506 0, NULL, NULL, NULL, NULL, NULL, 0);
507
508 vfs_mountroot(); /* Mount the root file system */
509 errorq_init(); /* after vfs_mountroot() so DDI root is ready */
510 cpu_kstat_init(CPU); /* after vfs_mountroot() so TOD is valid */
511 ddi_walk_devs(ddi_root_node(), pm_adjust_timestamps, NULL);
512 /* after vfs_mountroot() so hrestime is valid */
513
514 post_startup();
515 swaploaded = 1;
516
517 /*
518 * Initialize Solaris Audit Subsystem
519 */
520 audit_init();
521
522 /*
523 * Start the periodic hash rescale for all vmem arenas before we load
524 * protocol modules and drivers via strplumb() below. Some drivers
525 * might rely on heavy vmem operations that could hurt performance
526 * without the rescale.
527 */
528 vmem_update(NULL);
529
530 /*
531 * Plumb the protocol modules and drivers only if we are not
532 * networked booted, in this case we already did it in rootconf().
533 */
534 if (netboot == 0 && iscsiboot_prop == NULL)
535 (void) strplumb();
536
537 gethrestime(&PTOU(curproc)->u_start);
538 curthread->t_start = PTOU(curproc)->u_start.tv_sec;
539 p->p_mstart = gethrtime();
540
541 /*
542 * Perform setup functions that can only be done after root
543 * and swap have been set up.
544 */
545 consconfig();
546 #ifndef __sparc
547 release_bootstrap();
548 #endif
549
550 /*
551 * attach drivers with ddi-forceattach prop
552 * It must be done early enough to load hotplug drivers (e.g.
553 * pcmcia nexus) so that devices enumerated via hotplug is
554 * available before I/O subsystem is fully initialized.
555 */
556 i_ddi_forceattach_drivers();
557
558 /*
559 * Set the scan rate and other parameters of the paging subsystem.
560 */
561 setupclock();
562
563 /*
564 * Initialize process 0's lwp directory and lwpid hash table.
565 */
566 p->p_lwpdir = p->p_lwpfree = p0_lwpdir;
567 p->p_lwpdir->ld_next = p->p_lwpdir + 1;
568 p->p_lwpdir_sz = 2;
569 p->p_tidhash = p0_tidhash;
570 p->p_tidhash_sz = 2;
571 p0_lep.le_thread = curthread;
572 p0_lep.le_lwpid = curthread->t_tid;
573 p0_lep.le_start = curthread->t_start;
574 lwp_hash_in(p, &p0_lep, p0_tidhash, 2, 0);
575
576 /*
577 * Initialize extended accounting.
578 */
579 exacct_init();
580
581 /*
582 * Initialize threads of sysevent event channels
583 */
584 sysevent_evc_thrinit();
585
586 /*
587 * This must be done after post_startup() but before
588 * start_other_cpus()
589 */
590 lgrp_init(LGRP_INIT_STAGE4);
591
592 /*
593 * Perform MP initialization, if any.
594 */
595 start_other_cpus(0);
596
597 #ifdef __sparc
598 /*
599 * Release bootstrap here since PROM interfaces are
600 * used to start other CPUs above.
601 */
602 release_bootstrap();
603 #endif
604
605 /*
606 * Finish lgrp initialization after all CPUS are brought online.
607 */
608 lgrp_init(LGRP_INIT_STAGE5);
609
610 /*
611 * After mp_init(), number of cpus are known (this is
612 * true for the time being, when there are actually
613 * hot pluggable cpus then this scheme would not do).
614 * Any per cpu initialization is done here.
615 */
616 kmem_mp_init();
617
618 clock_tick_init_post();
619
620 for (initptr = &mp_init_tbl[0]; *initptr; initptr++)
621 (**initptr)();
622
623 /*
624 * These must be called after start_other_cpus
625 */
626 pm_cfb_setup_intr();
627 #if defined(__x86)
628 fastboot_post_startup();
629
630 smt_late_init();
631 #endif
632
633 /*
634 * Make init process; enter scheduling loop with system process.
635 *
636 * Note that we manually assign the pids for these processes, for
637 * historical reasons. If more pre-assigned pids are needed,
638 * FAMOUS_PIDS will have to be updated.
639 */
640
641 /* create init process */
642 if (newproc(start_init, NULL, defaultcid, 59, NULL,
643 FAMOUS_PID_INIT))
644 panic("main: unable to fork init.");
645
646 /* create pageout daemon */
647 if (newproc(pageout, NULL, syscid, maxclsyspri - 1, NULL,
648 FAMOUS_PID_PAGEOUT))
649 panic("main: unable to fork pageout()");
650
651 /* create fsflush daemon */
652 if (newproc(fsflush, NULL, syscid, minclsyspri, NULL,
653 FAMOUS_PID_FSFLUSH))
654 panic("main: unable to fork fsflush()");
655
656 /* create cluster process if we're a member of one */
657 if (cluster_bootflags & CLUSTER_BOOTED) {
658 if (newproc(cluster_wrapper, NULL, syscid, minclsyspri,
659 NULL, 0)) {
660 panic("main: unable to fork cluster()");
661 }
662 }
663
664 /*
665 * Create system threads (threads are associated with p0)
666 */
667
668 /* create module uninstall daemon */
669 /* BugID 1132273. If swapping over NFS need a bigger stack */
670 (void) thread_create(NULL, 0, (void (*)())mod_uninstall_daemon,
671 NULL, 0, &p0, TS_RUN, minclsyspri);
672
673 (void) thread_create(NULL, 0, seg_pasync_thread,
674 NULL, 0, &p0, TS_RUN, minclsyspri);
675
676 pid_setmin();
677
678 /* system is now ready */
679 mutex_exit(&ualock);
680
681 bcopy("sched", PTOU(curproc)->u_psargs, 6);
682 bcopy("sched", PTOU(curproc)->u_comm, 5);
683 sched();
684 /* NOTREACHED */
685 }
686