xref: /linux/init/main.c (revision a4eb44a6435d6d8f9e642407a4a06f65eb90ca04)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/init/main.c
4  *
5  *  Copyright (C) 1991, 1992  Linus Torvalds
6  *
7  *  GK 2/5/95  -  Changed to support mounting root fs via NFS
8  *  Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
9  *  Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
10  *  Simplified starting of init:  Michael A. Griffith <grif@acm.org>
11  */
12 
13 #define DEBUG		/* Enable initcall_debug */
14 
15 #include <linux/types.h>
16 #include <linux/extable.h>
17 #include <linux/module.h>
18 #include <linux/proc_fs.h>
19 #include <linux/binfmts.h>
20 #include <linux/kernel.h>
21 #include <linux/syscalls.h>
22 #include <linux/stackprotector.h>
23 #include <linux/string.h>
24 #include <linux/ctype.h>
25 #include <linux/delay.h>
26 #include <linux/ioport.h>
27 #include <linux/init.h>
28 #include <linux/initrd.h>
29 #include <linux/memblock.h>
30 #include <linux/acpi.h>
31 #include <linux/bootconfig.h>
32 #include <linux/console.h>
33 #include <linux/nmi.h>
34 #include <linux/percpu.h>
35 #include <linux/kmod.h>
36 #include <linux/kprobes.h>
37 #include <linux/vmalloc.h>
38 #include <linux/kernel_stat.h>
39 #include <linux/start_kernel.h>
40 #include <linux/security.h>
41 #include <linux/smp.h>
42 #include <linux/profile.h>
43 #include <linux/kfence.h>
44 #include <linux/rcupdate.h>
45 #include <linux/srcu.h>
46 #include <linux/moduleparam.h>
47 #include <linux/kallsyms.h>
48 #include <linux/buildid.h>
49 #include <linux/writeback.h>
50 #include <linux/cpu.h>
51 #include <linux/cpuset.h>
52 #include <linux/cgroup.h>
53 #include <linux/efi.h>
54 #include <linux/tick.h>
55 #include <linux/sched/isolation.h>
56 #include <linux/interrupt.h>
57 #include <linux/taskstats_kern.h>
58 #include <linux/delayacct.h>
59 #include <linux/unistd.h>
60 #include <linux/utsname.h>
61 #include <linux/rmap.h>
62 #include <linux/mempolicy.h>
63 #include <linux/key.h>
64 #include <linux/page_ext.h>
65 #include <linux/debug_locks.h>
66 #include <linux/debugobjects.h>
67 #include <linux/lockdep.h>
68 #include <linux/kmemleak.h>
69 #include <linux/padata.h>
70 #include <linux/pid_namespace.h>
71 #include <linux/device/driver.h>
72 #include <linux/kthread.h>
73 #include <linux/sched.h>
74 #include <linux/sched/init.h>
75 #include <linux/signal.h>
76 #include <linux/idr.h>
77 #include <linux/kgdb.h>
78 #include <linux/ftrace.h>
79 #include <linux/async.h>
80 #include <linux/shmem_fs.h>
81 #include <linux/slab.h>
82 #include <linux/perf_event.h>
83 #include <linux/ptrace.h>
84 #include <linux/pti.h>
85 #include <linux/blkdev.h>
86 #include <linux/sched/clock.h>
87 #include <linux/sched/task.h>
88 #include <linux/sched/task_stack.h>
89 #include <linux/context_tracking.h>
90 #include <linux/random.h>
91 #include <linux/list.h>
92 #include <linux/integrity.h>
93 #include <linux/proc_ns.h>
94 #include <linux/io.h>
95 #include <linux/cache.h>
96 #include <linux/rodata_test.h>
97 #include <linux/jump_label.h>
98 #include <linux/mem_encrypt.h>
99 #include <linux/kcsan.h>
100 #include <linux/init_syscalls.h>
101 #include <linux/stackdepot.h>
102 
103 #include <asm/io.h>
104 #include <asm/bugs.h>
105 #include <asm/setup.h>
106 #include <asm/sections.h>
107 #include <asm/cacheflush.h>
108 
109 #define CREATE_TRACE_POINTS
110 #include <trace/events/initcall.h>
111 
112 #include <kunit/test.h>
113 
114 static int kernel_init(void *);
115 
116 extern void init_IRQ(void);
117 extern void radix_tree_init(void);
118 
119 /*
120  * Debug helper: via this flag we know that we are in 'early bootup code'
121  * where only the boot processor is running with IRQ disabled.  This means
122  * two things - IRQ must not be enabled before the flag is cleared and some
123  * operations which are not allowed with IRQ disabled are allowed while the
124  * flag is set.
125  */
126 bool early_boot_irqs_disabled __read_mostly;
127 
128 enum system_states system_state __read_mostly;
129 EXPORT_SYMBOL(system_state);
130 
131 /*
132  * Boot command-line arguments
133  */
134 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
135 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
136 
137 extern void time_init(void);
138 /* Default late time init is NULL. archs can override this later. */
139 void (*__initdata late_time_init)(void);
140 
141 /* Untouched command line saved by arch-specific code. */
142 char __initdata boot_command_line[COMMAND_LINE_SIZE];
143 /* Untouched saved command line (eg. for /proc) */
144 char *saved_command_line;
145 /* Command line for parameter parsing */
146 static char *static_command_line;
147 /* Untouched extra command line */
148 static char *extra_command_line;
149 /* Extra init arguments */
150 static char *extra_init_args;
151 
152 #ifdef CONFIG_BOOT_CONFIG
153 /* Is bootconfig on command line? */
154 static bool bootconfig_found;
155 static size_t initargs_offs;
156 #else
157 # define bootconfig_found false
158 # define initargs_offs 0
159 #endif
160 
161 static char *execute_command;
162 static char *ramdisk_execute_command = "/init";
163 
164 /*
165  * Used to generate warnings if static_key manipulation functions are used
166  * before jump_label_init is called.
167  */
168 bool static_key_initialized __read_mostly;
169 EXPORT_SYMBOL_GPL(static_key_initialized);
170 
171 /*
172  * If set, this is an indication to the drivers that reset the underlying
173  * device before going ahead with the initialization otherwise driver might
174  * rely on the BIOS and skip the reset operation.
175  *
176  * This is useful if kernel is booting in an unreliable environment.
177  * For ex. kdump situation where previous kernel has crashed, BIOS has been
178  * skipped and devices will be in unknown state.
179  */
180 unsigned int reset_devices;
181 EXPORT_SYMBOL(reset_devices);
182 
183 static int __init set_reset_devices(char *str)
184 {
185 	reset_devices = 1;
186 	return 1;
187 }
188 
189 __setup("reset_devices", set_reset_devices);
190 
191 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
192 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
193 static const char *panic_later, *panic_param;
194 
195 extern const struct obs_kernel_param __setup_start[], __setup_end[];
196 
197 static bool __init obsolete_checksetup(char *line)
198 {
199 	const struct obs_kernel_param *p;
200 	bool had_early_param = false;
201 
202 	p = __setup_start;
203 	do {
204 		int n = strlen(p->str);
205 		if (parameqn(line, p->str, n)) {
206 			if (p->early) {
207 				/* Already done in parse_early_param?
208 				 * (Needs exact match on param part).
209 				 * Keep iterating, as we can have early
210 				 * params and __setups of same names 8( */
211 				if (line[n] == '\0' || line[n] == '=')
212 					had_early_param = true;
213 			} else if (!p->setup_func) {
214 				pr_warn("Parameter %s is obsolete, ignored\n",
215 					p->str);
216 				return true;
217 			} else if (p->setup_func(line + n))
218 				return true;
219 		}
220 		p++;
221 	} while (p < __setup_end);
222 
223 	return had_early_param;
224 }
225 
226 /*
227  * This should be approx 2 Bo*oMips to start (note initial shift), and will
228  * still work even if initially too large, it will just take slightly longer
229  */
230 unsigned long loops_per_jiffy = (1<<12);
231 EXPORT_SYMBOL(loops_per_jiffy);
232 
233 static int __init debug_kernel(char *str)
234 {
235 	console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
236 	return 0;
237 }
238 
239 static int __init quiet_kernel(char *str)
240 {
241 	console_loglevel = CONSOLE_LOGLEVEL_QUIET;
242 	return 0;
243 }
244 
245 early_param("debug", debug_kernel);
246 early_param("quiet", quiet_kernel);
247 
248 static int __init loglevel(char *str)
249 {
250 	int newlevel;
251 
252 	/*
253 	 * Only update loglevel value when a correct setting was passed,
254 	 * to prevent blind crashes (when loglevel being set to 0) that
255 	 * are quite hard to debug
256 	 */
257 	if (get_option(&str, &newlevel)) {
258 		console_loglevel = newlevel;
259 		return 0;
260 	}
261 
262 	return -EINVAL;
263 }
264 
265 early_param("loglevel", loglevel);
266 
267 #ifdef CONFIG_BLK_DEV_INITRD
268 static void * __init get_boot_config_from_initrd(u32 *_size, u32 *_csum)
269 {
270 	u32 size, csum;
271 	char *data;
272 	u32 *hdr;
273 	int i;
274 
275 	if (!initrd_end)
276 		return NULL;
277 
278 	data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
279 	/*
280 	 * Since Grub may align the size of initrd to 4, we must
281 	 * check the preceding 3 bytes as well.
282 	 */
283 	for (i = 0; i < 4; i++) {
284 		if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
285 			goto found;
286 		data--;
287 	}
288 	return NULL;
289 
290 found:
291 	hdr = (u32 *)(data - 8);
292 	size = le32_to_cpu(hdr[0]);
293 	csum = le32_to_cpu(hdr[1]);
294 
295 	data = ((void *)hdr) - size;
296 	if ((unsigned long)data < initrd_start) {
297 		pr_err("bootconfig size %d is greater than initrd size %ld\n",
298 			size, initrd_end - initrd_start);
299 		return NULL;
300 	}
301 
302 	/* Remove bootconfig from initramfs/initrd */
303 	initrd_end = (unsigned long)data;
304 	if (_size)
305 		*_size = size;
306 	if (_csum)
307 		*_csum = csum;
308 
309 	return data;
310 }
311 #else
312 static void * __init get_boot_config_from_initrd(u32 *_size, u32 *_csum)
313 {
314 	return NULL;
315 }
316 #endif
317 
318 #ifdef CONFIG_BOOT_CONFIG
319 
320 static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
321 
322 #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
323 
324 static int __init xbc_snprint_cmdline(char *buf, size_t size,
325 				      struct xbc_node *root)
326 {
327 	struct xbc_node *knode, *vnode;
328 	char *end = buf + size;
329 	const char *val;
330 	int ret;
331 
332 	xbc_node_for_each_key_value(root, knode, val) {
333 		ret = xbc_node_compose_key_after(root, knode,
334 					xbc_namebuf, XBC_KEYLEN_MAX);
335 		if (ret < 0)
336 			return ret;
337 
338 		vnode = xbc_node_get_child(knode);
339 		if (!vnode) {
340 			ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
341 			if (ret < 0)
342 				return ret;
343 			buf += ret;
344 			continue;
345 		}
346 		xbc_array_for_each_value(vnode, val) {
347 			ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ",
348 				       xbc_namebuf, val);
349 			if (ret < 0)
350 				return ret;
351 			buf += ret;
352 		}
353 	}
354 
355 	return buf - (end - size);
356 }
357 #undef rest
358 
359 /* Make an extra command line under given key word */
360 static char * __init xbc_make_cmdline(const char *key)
361 {
362 	struct xbc_node *root;
363 	char *new_cmdline;
364 	int ret, len = 0;
365 
366 	root = xbc_find_node(key);
367 	if (!root)
368 		return NULL;
369 
370 	/* Count required buffer size */
371 	len = xbc_snprint_cmdline(NULL, 0, root);
372 	if (len <= 0)
373 		return NULL;
374 
375 	new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
376 	if (!new_cmdline) {
377 		pr_err("Failed to allocate memory for extra kernel cmdline.\n");
378 		return NULL;
379 	}
380 
381 	ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
382 	if (ret < 0 || ret > len) {
383 		pr_err("Failed to print extra kernel cmdline.\n");
384 		memblock_free(new_cmdline, len + 1);
385 		return NULL;
386 	}
387 
388 	return new_cmdline;
389 }
390 
391 static int __init bootconfig_params(char *param, char *val,
392 				    const char *unused, void *arg)
393 {
394 	if (strcmp(param, "bootconfig") == 0) {
395 		bootconfig_found = true;
396 	}
397 	return 0;
398 }
399 
400 static int __init warn_bootconfig(char *str)
401 {
402 	/* The 'bootconfig' has been handled by bootconfig_params(). */
403 	return 0;
404 }
405 
406 static void __init setup_boot_config(void)
407 {
408 	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
409 	const char *msg;
410 	int pos;
411 	u32 size, csum;
412 	char *data, *err;
413 	int ret;
414 
415 	/* Cut out the bootconfig data even if we have no bootconfig option */
416 	data = get_boot_config_from_initrd(&size, &csum);
417 
418 	strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
419 	err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
420 			 bootconfig_params);
421 
422 	if (IS_ERR(err) || !bootconfig_found)
423 		return;
424 
425 	/* parse_args() stops at the next param of '--' and returns an address */
426 	if (err)
427 		initargs_offs = err - tmp_cmdline;
428 
429 	if (!data) {
430 		pr_err("'bootconfig' found on command line, but no bootconfig found\n");
431 		return;
432 	}
433 
434 	if (size >= XBC_DATA_MAX) {
435 		pr_err("bootconfig size %d greater than max size %d\n",
436 			size, XBC_DATA_MAX);
437 		return;
438 	}
439 
440 	if (xbc_calc_checksum(data, size) != csum) {
441 		pr_err("bootconfig checksum failed\n");
442 		return;
443 	}
444 
445 	ret = xbc_init(data, size, &msg, &pos);
446 	if (ret < 0) {
447 		if (pos < 0)
448 			pr_err("Failed to init bootconfig: %s.\n", msg);
449 		else
450 			pr_err("Failed to parse bootconfig: %s at %d.\n",
451 				msg, pos);
452 	} else {
453 		xbc_get_info(&ret, NULL);
454 		pr_info("Load bootconfig: %d bytes %d nodes\n", size, ret);
455 		/* keys starting with "kernel." are passed via cmdline */
456 		extra_command_line = xbc_make_cmdline("kernel");
457 		/* Also, "init." keys are init arguments */
458 		extra_init_args = xbc_make_cmdline("init");
459 	}
460 	return;
461 }
462 
463 static void __init exit_boot_config(void)
464 {
465 	xbc_exit();
466 }
467 
468 #else	/* !CONFIG_BOOT_CONFIG */
469 
470 static void __init setup_boot_config(void)
471 {
472 	/* Remove bootconfig data from initrd */
473 	get_boot_config_from_initrd(NULL, NULL);
474 }
475 
476 static int __init warn_bootconfig(char *str)
477 {
478 	pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
479 	return 0;
480 }
481 
482 #define exit_boot_config()	do {} while (0)
483 
484 #endif	/* CONFIG_BOOT_CONFIG */
485 
486 early_param("bootconfig", warn_bootconfig);
487 
488 /* Change NUL term back to "=", to make "param" the whole string. */
489 static void __init repair_env_string(char *param, char *val)
490 {
491 	if (val) {
492 		/* param=val or param="val"? */
493 		if (val == param+strlen(param)+1)
494 			val[-1] = '=';
495 		else if (val == param+strlen(param)+2) {
496 			val[-2] = '=';
497 			memmove(val-1, val, strlen(val)+1);
498 		} else
499 			BUG();
500 	}
501 }
502 
503 /* Anything after -- gets handed straight to init. */
504 static int __init set_init_arg(char *param, char *val,
505 			       const char *unused, void *arg)
506 {
507 	unsigned int i;
508 
509 	if (panic_later)
510 		return 0;
511 
512 	repair_env_string(param, val);
513 
514 	for (i = 0; argv_init[i]; i++) {
515 		if (i == MAX_INIT_ARGS) {
516 			panic_later = "init";
517 			panic_param = param;
518 			return 0;
519 		}
520 	}
521 	argv_init[i] = param;
522 	return 0;
523 }
524 
525 /*
526  * Unknown boot options get handed to init, unless they look like
527  * unused parameters (modprobe will find them in /proc/cmdline).
528  */
529 static int __init unknown_bootoption(char *param, char *val,
530 				     const char *unused, void *arg)
531 {
532 	size_t len = strlen(param);
533 
534 	repair_env_string(param, val);
535 
536 	/* Handle obsolete-style parameters */
537 	if (obsolete_checksetup(param))
538 		return 0;
539 
540 	/* Unused module parameter. */
541 	if (strnchr(param, len, '.'))
542 		return 0;
543 
544 	if (panic_later)
545 		return 0;
546 
547 	if (val) {
548 		/* Environment option */
549 		unsigned int i;
550 		for (i = 0; envp_init[i]; i++) {
551 			if (i == MAX_INIT_ENVS) {
552 				panic_later = "env";
553 				panic_param = param;
554 			}
555 			if (!strncmp(param, envp_init[i], len+1))
556 				break;
557 		}
558 		envp_init[i] = param;
559 	} else {
560 		/* Command line option */
561 		unsigned int i;
562 		for (i = 0; argv_init[i]; i++) {
563 			if (i == MAX_INIT_ARGS) {
564 				panic_later = "init";
565 				panic_param = param;
566 			}
567 		}
568 		argv_init[i] = param;
569 	}
570 	return 0;
571 }
572 
573 static int __init init_setup(char *str)
574 {
575 	unsigned int i;
576 
577 	execute_command = str;
578 	/*
579 	 * In case LILO is going to boot us with default command line,
580 	 * it prepends "auto" before the whole cmdline which makes
581 	 * the shell think it should execute a script with such name.
582 	 * So we ignore all arguments entered _before_ init=... [MJ]
583 	 */
584 	for (i = 1; i < MAX_INIT_ARGS; i++)
585 		argv_init[i] = NULL;
586 	return 1;
587 }
588 __setup("init=", init_setup);
589 
590 static int __init rdinit_setup(char *str)
591 {
592 	unsigned int i;
593 
594 	ramdisk_execute_command = str;
595 	/* See "auto" comment in init_setup */
596 	for (i = 1; i < MAX_INIT_ARGS; i++)
597 		argv_init[i] = NULL;
598 	return 1;
599 }
600 __setup("rdinit=", rdinit_setup);
601 
602 #ifndef CONFIG_SMP
603 static const unsigned int setup_max_cpus = NR_CPUS;
604 static inline void setup_nr_cpu_ids(void) { }
605 static inline void smp_prepare_cpus(unsigned int maxcpus) { }
606 #endif
607 
608 /*
609  * We need to store the untouched command line for future reference.
610  * We also need to store the touched command line since the parameter
611  * parsing is performed in place, and we should allow a component to
612  * store reference of name/value for future reference.
613  */
614 static void __init setup_command_line(char *command_line)
615 {
616 	size_t len, xlen = 0, ilen = 0;
617 
618 	if (extra_command_line)
619 		xlen = strlen(extra_command_line);
620 	if (extra_init_args)
621 		ilen = strlen(extra_init_args) + 4; /* for " -- " */
622 
623 	len = xlen + strlen(boot_command_line) + 1;
624 
625 	saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES);
626 	if (!saved_command_line)
627 		panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen);
628 
629 	static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
630 	if (!static_command_line)
631 		panic("%s: Failed to allocate %zu bytes\n", __func__, len);
632 
633 	if (xlen) {
634 		/*
635 		 * We have to put extra_command_line before boot command
636 		 * lines because there could be dashes (separator of init
637 		 * command line) in the command lines.
638 		 */
639 		strcpy(saved_command_line, extra_command_line);
640 		strcpy(static_command_line, extra_command_line);
641 	}
642 	strcpy(saved_command_line + xlen, boot_command_line);
643 	strcpy(static_command_line + xlen, command_line);
644 
645 	if (ilen) {
646 		/*
647 		 * Append supplemental init boot args to saved_command_line
648 		 * so that user can check what command line options passed
649 		 * to init.
650 		 * The order should always be
651 		 * " -- "[bootconfig init-param][cmdline init-param]
652 		 */
653 		if (initargs_offs) {
654 			len = xlen + initargs_offs;
655 			strcpy(saved_command_line + len, extra_init_args);
656 			len += ilen - 4;	/* strlen(extra_init_args) */
657 			strcpy(saved_command_line + len,
658 				boot_command_line + initargs_offs - 1);
659 		} else {
660 			len = strlen(saved_command_line);
661 			strcpy(saved_command_line + len, " -- ");
662 			len += 4;
663 			strcpy(saved_command_line + len, extra_init_args);
664 		}
665 	}
666 }
667 
668 /*
669  * We need to finalize in a non-__init function or else race conditions
670  * between the root thread and the init thread may cause start_kernel to
671  * be reaped by free_initmem before the root thread has proceeded to
672  * cpu_idle.
673  *
674  * gcc-3.4 accidentally inlines this function, so use noinline.
675  */
676 
677 static __initdata DECLARE_COMPLETION(kthreadd_done);
678 
679 noinline void __ref rest_init(void)
680 {
681 	struct task_struct *tsk;
682 	int pid;
683 
684 	rcu_scheduler_starting();
685 	/*
686 	 * We need to spawn init first so that it obtains pid 1, however
687 	 * the init task will end up wanting to create kthreads, which, if
688 	 * we schedule it before we create kthreadd, will OOPS.
689 	 */
690 	pid = kernel_thread(kernel_init, NULL, CLONE_FS);
691 	/*
692 	 * Pin init on the boot CPU. Task migration is not properly working
693 	 * until sched_init_smp() has been run. It will set the allowed
694 	 * CPUs for init to the non isolated CPUs.
695 	 */
696 	rcu_read_lock();
697 	tsk = find_task_by_pid_ns(pid, &init_pid_ns);
698 	tsk->flags |= PF_NO_SETAFFINITY;
699 	set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
700 	rcu_read_unlock();
701 
702 	numa_default_policy();
703 	pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
704 	rcu_read_lock();
705 	kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
706 	rcu_read_unlock();
707 
708 	/*
709 	 * Enable might_sleep() and smp_processor_id() checks.
710 	 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
711 	 * kernel_thread() would trigger might_sleep() splats. With
712 	 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
713 	 * already, but it's stuck on the kthreadd_done completion.
714 	 */
715 	system_state = SYSTEM_SCHEDULING;
716 
717 	complete(&kthreadd_done);
718 
719 	/*
720 	 * The boot idle thread must execute schedule()
721 	 * at least once to get things moving:
722 	 */
723 	schedule_preempt_disabled();
724 	/* Call into cpu_idle with preempt disabled */
725 	cpu_startup_entry(CPUHP_ONLINE);
726 }
727 
728 /* Check for early params. */
729 static int __init do_early_param(char *param, char *val,
730 				 const char *unused, void *arg)
731 {
732 	const struct obs_kernel_param *p;
733 
734 	for (p = __setup_start; p < __setup_end; p++) {
735 		if ((p->early && parameq(param, p->str)) ||
736 		    (strcmp(param, "console") == 0 &&
737 		     strcmp(p->str, "earlycon") == 0)
738 		) {
739 			if (p->setup_func(val) != 0)
740 				pr_warn("Malformed early option '%s'\n", param);
741 		}
742 	}
743 	/* We accept everything at this stage. */
744 	return 0;
745 }
746 
747 void __init parse_early_options(char *cmdline)
748 {
749 	parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
750 		   do_early_param);
751 }
752 
753 /* Arch code calls this early on, or if not, just before other parsing. */
754 void __init parse_early_param(void)
755 {
756 	static int done __initdata;
757 	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
758 
759 	if (done)
760 		return;
761 
762 	/* All fall through to do_early_param. */
763 	strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
764 	parse_early_options(tmp_cmdline);
765 	done = 1;
766 }
767 
768 void __init __weak arch_post_acpi_subsys_init(void) { }
769 
770 void __init __weak smp_setup_processor_id(void)
771 {
772 }
773 
774 # if THREAD_SIZE >= PAGE_SIZE
775 void __init __weak thread_stack_cache_init(void)
776 {
777 }
778 #endif
779 
780 void __init __weak mem_encrypt_init(void) { }
781 
782 void __init __weak poking_init(void) { }
783 
784 void __init __weak pgtable_cache_init(void) { }
785 
786 void __init __weak trap_init(void) { }
787 
788 bool initcall_debug;
789 core_param(initcall_debug, initcall_debug, bool, 0644);
790 
791 #ifdef TRACEPOINTS_ENABLED
792 static void __init initcall_debug_enable(void);
793 #else
794 static inline void initcall_debug_enable(void)
795 {
796 }
797 #endif
798 
799 /* Report memory auto-initialization states for this boot. */
800 static void __init report_meminit(void)
801 {
802 	const char *stack;
803 
804 	if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))
805 		stack = "all(pattern)";
806 	else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))
807 		stack = "all(zero)";
808 	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))
809 		stack = "byref_all(zero)";
810 	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))
811 		stack = "byref(zero)";
812 	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))
813 		stack = "__user(zero)";
814 	else
815 		stack = "off";
816 
817 	pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
818 		stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
819 		want_init_on_free() ? "on" : "off");
820 	if (want_init_on_free())
821 		pr_info("mem auto-init: clearing system memory may take some time...\n");
822 }
823 
824 /*
825  * Set up kernel memory allocators
826  */
827 static void __init mm_init(void)
828 {
829 	/*
830 	 * page_ext requires contiguous pages,
831 	 * bigger than MAX_ORDER unless SPARSEMEM.
832 	 */
833 	page_ext_init_flatmem();
834 	init_mem_debugging_and_hardening();
835 	kfence_alloc_pool();
836 	report_meminit();
837 	stack_depot_early_init();
838 	mem_init();
839 	mem_init_print_info();
840 	kmem_cache_init();
841 	/*
842 	 * page_owner must be initialized after buddy is ready, and also after
843 	 * slab is ready so that stack_depot_init() works properly
844 	 */
845 	page_ext_init_flatmem_late();
846 	kmemleak_init();
847 	pgtable_init();
848 	debug_objects_mem_init();
849 	vmalloc_init();
850 	/* Should be run before the first non-init thread is created */
851 	init_espfix_bsp();
852 	/* Should be run after espfix64 is set up. */
853 	pti_init();
854 }
855 
856 #ifdef CONFIG_HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET
857 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
858 			   randomize_kstack_offset);
859 DEFINE_PER_CPU(u32, kstack_offset);
860 
861 static int __init early_randomize_kstack_offset(char *buf)
862 {
863 	int ret;
864 	bool bool_result;
865 
866 	ret = kstrtobool(buf, &bool_result);
867 	if (ret)
868 		return ret;
869 
870 	if (bool_result)
871 		static_branch_enable(&randomize_kstack_offset);
872 	else
873 		static_branch_disable(&randomize_kstack_offset);
874 	return 0;
875 }
876 early_param("randomize_kstack_offset", early_randomize_kstack_offset);
877 #endif
878 
879 void __init __weak arch_call_rest_init(void)
880 {
881 	rest_init();
882 }
883 
884 static void __init print_unknown_bootoptions(void)
885 {
886 	char *unknown_options;
887 	char *end;
888 	const char *const *p;
889 	size_t len;
890 
891 	if (panic_later || (!argv_init[1] && !envp_init[2]))
892 		return;
893 
894 	/*
895 	 * Determine how many options we have to print out, plus a space
896 	 * before each
897 	 */
898 	len = 1; /* null terminator */
899 	for (p = &argv_init[1]; *p; p++) {
900 		len++;
901 		len += strlen(*p);
902 	}
903 	for (p = &envp_init[2]; *p; p++) {
904 		len++;
905 		len += strlen(*p);
906 	}
907 
908 	unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
909 	if (!unknown_options) {
910 		pr_err("%s: Failed to allocate %zu bytes\n",
911 			__func__, len);
912 		return;
913 	}
914 	end = unknown_options;
915 
916 	for (p = &argv_init[1]; *p; p++)
917 		end += sprintf(end, " %s", *p);
918 	for (p = &envp_init[2]; *p; p++)
919 		end += sprintf(end, " %s", *p);
920 
921 	/* Start at unknown_options[1] to skip the initial space */
922 	pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
923 		&unknown_options[1]);
924 	memblock_free(unknown_options, len);
925 }
926 
927 asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
928 {
929 	char *command_line;
930 	char *after_dashes;
931 
932 	set_task_stack_end_magic(&init_task);
933 	smp_setup_processor_id();
934 	debug_objects_early_init();
935 	init_vmlinux_build_id();
936 
937 	cgroup_init_early();
938 
939 	local_irq_disable();
940 	early_boot_irqs_disabled = true;
941 
942 	/*
943 	 * Interrupts are still disabled. Do necessary setups, then
944 	 * enable them.
945 	 */
946 	boot_cpu_init();
947 	page_address_init();
948 	pr_notice("%s", linux_banner);
949 	early_security_init();
950 	setup_arch(&command_line);
951 	setup_boot_config();
952 	setup_command_line(command_line);
953 	setup_nr_cpu_ids();
954 	setup_per_cpu_areas();
955 	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
956 	boot_cpu_hotplug_init();
957 
958 	build_all_zonelists(NULL);
959 	page_alloc_init();
960 
961 	pr_notice("Kernel command line: %s\n", saved_command_line);
962 	/* parameters may set static keys */
963 	jump_label_init();
964 	parse_early_param();
965 	after_dashes = parse_args("Booting kernel",
966 				  static_command_line, __start___param,
967 				  __stop___param - __start___param,
968 				  -1, -1, NULL, &unknown_bootoption);
969 	print_unknown_bootoptions();
970 	if (!IS_ERR_OR_NULL(after_dashes))
971 		parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
972 			   NULL, set_init_arg);
973 	if (extra_init_args)
974 		parse_args("Setting extra init args", extra_init_args,
975 			   NULL, 0, -1, -1, NULL, set_init_arg);
976 
977 	/*
978 	 * These use large bootmem allocations and must precede
979 	 * kmem_cache_init()
980 	 */
981 	setup_log_buf(0);
982 	vfs_caches_init_early();
983 	sort_main_extable();
984 	trap_init();
985 	mm_init();
986 
987 	ftrace_init();
988 
989 	/* trace_printk can be enabled here */
990 	early_trace_init();
991 
992 	/*
993 	 * Set up the scheduler prior starting any interrupts (such as the
994 	 * timer interrupt). Full topology setup happens at smp_init()
995 	 * time - but meanwhile we still have a functioning scheduler.
996 	 */
997 	sched_init();
998 
999 	if (WARN(!irqs_disabled(),
1000 		 "Interrupts were enabled *very* early, fixing it\n"))
1001 		local_irq_disable();
1002 	radix_tree_init();
1003 
1004 	/*
1005 	 * Set up housekeeping before setting up workqueues to allow the unbound
1006 	 * workqueue to take non-housekeeping into account.
1007 	 */
1008 	housekeeping_init();
1009 
1010 	/*
1011 	 * Allow workqueue creation and work item queueing/cancelling
1012 	 * early.  Work item execution depends on kthreads and starts after
1013 	 * workqueue_init().
1014 	 */
1015 	workqueue_init_early();
1016 
1017 	rcu_init();
1018 
1019 	/* Trace events are available after this */
1020 	trace_init();
1021 
1022 	if (initcall_debug)
1023 		initcall_debug_enable();
1024 
1025 	context_tracking_init();
1026 	/* init some links before init_ISA_irqs() */
1027 	early_irq_init();
1028 	init_IRQ();
1029 	tick_init();
1030 	rcu_init_nohz();
1031 	init_timers();
1032 	srcu_init();
1033 	hrtimers_init();
1034 	softirq_init();
1035 	timekeeping_init();
1036 	kfence_init();
1037 
1038 	/*
1039 	 * For best initial stack canary entropy, prepare it after:
1040 	 * - setup_arch() for any UEFI RNG entropy and boot cmdline access
1041 	 * - timekeeping_init() for ktime entropy used in rand_initialize()
1042 	 * - rand_initialize() to get any arch-specific entropy like RDRAND
1043 	 * - add_latent_entropy() to get any latent entropy
1044 	 * - adding command line entropy
1045 	 */
1046 	rand_initialize();
1047 	add_latent_entropy();
1048 	add_device_randomness(command_line, strlen(command_line));
1049 	boot_init_stack_canary();
1050 
1051 	time_init();
1052 	perf_event_init();
1053 	profile_init();
1054 	call_function_init();
1055 	WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1056 
1057 	early_boot_irqs_disabled = false;
1058 	local_irq_enable();
1059 
1060 	kmem_cache_init_late();
1061 
1062 	/*
1063 	 * HACK ALERT! This is early. We're enabling the console before
1064 	 * we've done PCI setups etc, and console_init() must be aware of
1065 	 * this. But we do want output early, in case something goes wrong.
1066 	 */
1067 	console_init();
1068 	if (panic_later)
1069 		panic("Too many boot %s vars at `%s'", panic_later,
1070 		      panic_param);
1071 
1072 	lockdep_init();
1073 
1074 	/*
1075 	 * Need to run this when irqs are enabled, because it wants
1076 	 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1077 	 * too:
1078 	 */
1079 	locking_selftest();
1080 
1081 	/*
1082 	 * This needs to be called before any devices perform DMA
1083 	 * operations that might use the SWIOTLB bounce buffers. It will
1084 	 * mark the bounce buffers as decrypted so that their usage will
1085 	 * not cause "plain-text" data to be decrypted when accessed.
1086 	 */
1087 	mem_encrypt_init();
1088 
1089 #ifdef CONFIG_BLK_DEV_INITRD
1090 	if (initrd_start && !initrd_below_start_ok &&
1091 	    page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1092 		pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1093 		    page_to_pfn(virt_to_page((void *)initrd_start)),
1094 		    min_low_pfn);
1095 		initrd_start = 0;
1096 	}
1097 #endif
1098 	setup_per_cpu_pageset();
1099 	numa_policy_init();
1100 	acpi_early_init();
1101 	if (late_time_init)
1102 		late_time_init();
1103 	sched_clock_init();
1104 	calibrate_delay();
1105 	pid_idr_init();
1106 	anon_vma_init();
1107 #ifdef CONFIG_X86
1108 	if (efi_enabled(EFI_RUNTIME_SERVICES))
1109 		efi_enter_virtual_mode();
1110 #endif
1111 	thread_stack_cache_init();
1112 	cred_init();
1113 	fork_init();
1114 	proc_caches_init();
1115 	uts_ns_init();
1116 	key_init();
1117 	security_init();
1118 	dbg_late_init();
1119 	vfs_caches_init();
1120 	pagecache_init();
1121 	signals_init();
1122 	seq_file_init();
1123 	proc_root_init();
1124 	nsfs_init();
1125 	cpuset_init();
1126 	cgroup_init();
1127 	taskstats_init_early();
1128 	delayacct_init();
1129 
1130 	poking_init();
1131 	check_bugs();
1132 
1133 	acpi_subsystem_init();
1134 	arch_post_acpi_subsys_init();
1135 	kcsan_init();
1136 
1137 	/* Do the rest non-__init'ed, we're now alive */
1138 	arch_call_rest_init();
1139 
1140 	prevent_tail_call_optimization();
1141 }
1142 
1143 /* Call all constructor functions linked into the kernel. */
1144 static void __init do_ctors(void)
1145 {
1146 /*
1147  * For UML, the constructors have already been called by the
1148  * normal setup code as it's just a normal ELF binary, so we
1149  * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1150  * even on UML for modules.
1151  */
1152 #if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1153 	ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1154 
1155 	for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1156 		(*fn)();
1157 #endif
1158 }
1159 
1160 #ifdef CONFIG_KALLSYMS
1161 struct blacklist_entry {
1162 	struct list_head next;
1163 	char *buf;
1164 };
1165 
1166 static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1167 
1168 static int __init initcall_blacklist(char *str)
1169 {
1170 	char *str_entry;
1171 	struct blacklist_entry *entry;
1172 
1173 	/* str argument is a comma-separated list of functions */
1174 	do {
1175 		str_entry = strsep(&str, ",");
1176 		if (str_entry) {
1177 			pr_debug("blacklisting initcall %s\n", str_entry);
1178 			entry = memblock_alloc(sizeof(*entry),
1179 					       SMP_CACHE_BYTES);
1180 			if (!entry)
1181 				panic("%s: Failed to allocate %zu bytes\n",
1182 				      __func__, sizeof(*entry));
1183 			entry->buf = memblock_alloc(strlen(str_entry) + 1,
1184 						    SMP_CACHE_BYTES);
1185 			if (!entry->buf)
1186 				panic("%s: Failed to allocate %zu bytes\n",
1187 				      __func__, strlen(str_entry) + 1);
1188 			strcpy(entry->buf, str_entry);
1189 			list_add(&entry->next, &blacklisted_initcalls);
1190 		}
1191 	} while (str_entry);
1192 
1193 	return 0;
1194 }
1195 
1196 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1197 {
1198 	struct blacklist_entry *entry;
1199 	char fn_name[KSYM_SYMBOL_LEN];
1200 	unsigned long addr;
1201 
1202 	if (list_empty(&blacklisted_initcalls))
1203 		return false;
1204 
1205 	addr = (unsigned long) dereference_function_descriptor(fn);
1206 	sprint_symbol_no_offset(fn_name, addr);
1207 
1208 	/*
1209 	 * fn will be "function_name [module_name]" where [module_name] is not
1210 	 * displayed for built-in init functions.  Strip off the [module_name].
1211 	 */
1212 	strreplace(fn_name, ' ', '\0');
1213 
1214 	list_for_each_entry(entry, &blacklisted_initcalls, next) {
1215 		if (!strcmp(fn_name, entry->buf)) {
1216 			pr_debug("initcall %s blacklisted\n", fn_name);
1217 			return true;
1218 		}
1219 	}
1220 
1221 	return false;
1222 }
1223 #else
1224 static int __init initcall_blacklist(char *str)
1225 {
1226 	pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1227 	return 0;
1228 }
1229 
1230 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1231 {
1232 	return false;
1233 }
1234 #endif
1235 __setup("initcall_blacklist=", initcall_blacklist);
1236 
1237 static __init_or_module void
1238 trace_initcall_start_cb(void *data, initcall_t fn)
1239 {
1240 	ktime_t *calltime = (ktime_t *)data;
1241 
1242 	printk(KERN_DEBUG "calling  %pS @ %i\n", fn, task_pid_nr(current));
1243 	*calltime = ktime_get();
1244 }
1245 
1246 static __init_or_module void
1247 trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1248 {
1249 	ktime_t *calltime = (ktime_t *)data;
1250 	ktime_t delta, rettime;
1251 	unsigned long long duration;
1252 
1253 	rettime = ktime_get();
1254 	delta = ktime_sub(rettime, *calltime);
1255 	duration = (unsigned long long) ktime_to_ns(delta) >> 10;
1256 	printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1257 		 fn, ret, duration);
1258 }
1259 
1260 static ktime_t initcall_calltime;
1261 
1262 #ifdef TRACEPOINTS_ENABLED
1263 static void __init initcall_debug_enable(void)
1264 {
1265 	int ret;
1266 
1267 	ret = register_trace_initcall_start(trace_initcall_start_cb,
1268 					    &initcall_calltime);
1269 	ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1270 					      &initcall_calltime);
1271 	WARN(ret, "Failed to register initcall tracepoints\n");
1272 }
1273 # define do_trace_initcall_start	trace_initcall_start
1274 # define do_trace_initcall_finish	trace_initcall_finish
1275 #else
1276 static inline void do_trace_initcall_start(initcall_t fn)
1277 {
1278 	if (!initcall_debug)
1279 		return;
1280 	trace_initcall_start_cb(&initcall_calltime, fn);
1281 }
1282 static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1283 {
1284 	if (!initcall_debug)
1285 		return;
1286 	trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1287 }
1288 #endif /* !TRACEPOINTS_ENABLED */
1289 
1290 int __init_or_module do_one_initcall(initcall_t fn)
1291 {
1292 	int count = preempt_count();
1293 	char msgbuf[64];
1294 	int ret;
1295 
1296 	if (initcall_blacklisted(fn))
1297 		return -EPERM;
1298 
1299 	do_trace_initcall_start(fn);
1300 	ret = fn();
1301 	do_trace_initcall_finish(fn, ret);
1302 
1303 	msgbuf[0] = 0;
1304 
1305 	if (preempt_count() != count) {
1306 		sprintf(msgbuf, "preemption imbalance ");
1307 		preempt_count_set(count);
1308 	}
1309 	if (irqs_disabled()) {
1310 		strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1311 		local_irq_enable();
1312 	}
1313 	WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1314 
1315 	add_latent_entropy();
1316 	return ret;
1317 }
1318 
1319 
1320 extern initcall_entry_t __initcall_start[];
1321 extern initcall_entry_t __initcall0_start[];
1322 extern initcall_entry_t __initcall1_start[];
1323 extern initcall_entry_t __initcall2_start[];
1324 extern initcall_entry_t __initcall3_start[];
1325 extern initcall_entry_t __initcall4_start[];
1326 extern initcall_entry_t __initcall5_start[];
1327 extern initcall_entry_t __initcall6_start[];
1328 extern initcall_entry_t __initcall7_start[];
1329 extern initcall_entry_t __initcall_end[];
1330 
1331 static initcall_entry_t *initcall_levels[] __initdata = {
1332 	__initcall0_start,
1333 	__initcall1_start,
1334 	__initcall2_start,
1335 	__initcall3_start,
1336 	__initcall4_start,
1337 	__initcall5_start,
1338 	__initcall6_start,
1339 	__initcall7_start,
1340 	__initcall_end,
1341 };
1342 
1343 /* Keep these in sync with initcalls in include/linux/init.h */
1344 static const char *initcall_level_names[] __initdata = {
1345 	"pure",
1346 	"core",
1347 	"postcore",
1348 	"arch",
1349 	"subsys",
1350 	"fs",
1351 	"device",
1352 	"late",
1353 };
1354 
1355 static int __init ignore_unknown_bootoption(char *param, char *val,
1356 			       const char *unused, void *arg)
1357 {
1358 	return 0;
1359 }
1360 
1361 static void __init do_initcall_level(int level, char *command_line)
1362 {
1363 	initcall_entry_t *fn;
1364 
1365 	parse_args(initcall_level_names[level],
1366 		   command_line, __start___param,
1367 		   __stop___param - __start___param,
1368 		   level, level,
1369 		   NULL, ignore_unknown_bootoption);
1370 
1371 	trace_initcall_level(initcall_level_names[level]);
1372 	for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1373 		do_one_initcall(initcall_from_entry(fn));
1374 }
1375 
1376 static void __init do_initcalls(void)
1377 {
1378 	int level;
1379 	size_t len = strlen(saved_command_line) + 1;
1380 	char *command_line;
1381 
1382 	command_line = kzalloc(len, GFP_KERNEL);
1383 	if (!command_line)
1384 		panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1385 
1386 	for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1387 		/* Parser modifies command_line, restore it each time */
1388 		strcpy(command_line, saved_command_line);
1389 		do_initcall_level(level, command_line);
1390 	}
1391 
1392 	kfree(command_line);
1393 }
1394 
1395 /*
1396  * Ok, the machine is now initialized. None of the devices
1397  * have been touched yet, but the CPU subsystem is up and
1398  * running, and memory and process management works.
1399  *
1400  * Now we can finally start doing some real work..
1401  */
1402 static void __init do_basic_setup(void)
1403 {
1404 	cpuset_init_smp();
1405 	driver_init();
1406 	init_irq_proc();
1407 	do_ctors();
1408 	do_initcalls();
1409 }
1410 
1411 static void __init do_pre_smp_initcalls(void)
1412 {
1413 	initcall_entry_t *fn;
1414 
1415 	trace_initcall_level("early");
1416 	for (fn = __initcall_start; fn < __initcall0_start; fn++)
1417 		do_one_initcall(initcall_from_entry(fn));
1418 }
1419 
1420 static int run_init_process(const char *init_filename)
1421 {
1422 	const char *const *p;
1423 
1424 	argv_init[0] = init_filename;
1425 	pr_info("Run %s as init process\n", init_filename);
1426 	pr_debug("  with arguments:\n");
1427 	for (p = argv_init; *p; p++)
1428 		pr_debug("    %s\n", *p);
1429 	pr_debug("  with environment:\n");
1430 	for (p = envp_init; *p; p++)
1431 		pr_debug("    %s\n", *p);
1432 	return kernel_execve(init_filename, argv_init, envp_init);
1433 }
1434 
1435 static int try_to_run_init_process(const char *init_filename)
1436 {
1437 	int ret;
1438 
1439 	ret = run_init_process(init_filename);
1440 
1441 	if (ret && ret != -ENOENT) {
1442 		pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1443 		       init_filename, ret);
1444 	}
1445 
1446 	return ret;
1447 }
1448 
1449 static noinline void __init kernel_init_freeable(void);
1450 
1451 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1452 bool rodata_enabled __ro_after_init = true;
1453 static int __init set_debug_rodata(char *str)
1454 {
1455 	return strtobool(str, &rodata_enabled);
1456 }
1457 __setup("rodata=", set_debug_rodata);
1458 #endif
1459 
1460 #ifdef CONFIG_STRICT_KERNEL_RWX
1461 static void mark_readonly(void)
1462 {
1463 	if (rodata_enabled) {
1464 		/*
1465 		 * load_module() results in W+X mappings, which are cleaned
1466 		 * up with call_rcu().  Let's make sure that queued work is
1467 		 * flushed so that we don't hit false positives looking for
1468 		 * insecure pages which are W+X.
1469 		 */
1470 		rcu_barrier();
1471 		mark_rodata_ro();
1472 		rodata_test();
1473 	} else
1474 		pr_info("Kernel memory protection disabled.\n");
1475 }
1476 #elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)
1477 static inline void mark_readonly(void)
1478 {
1479 	pr_warn("Kernel memory protection not selected by kernel config.\n");
1480 }
1481 #else
1482 static inline void mark_readonly(void)
1483 {
1484 	pr_warn("This architecture does not have kernel memory protection.\n");
1485 }
1486 #endif
1487 
1488 void __weak free_initmem(void)
1489 {
1490 	free_initmem_default(POISON_FREE_INITMEM);
1491 }
1492 
1493 static int __ref kernel_init(void *unused)
1494 {
1495 	int ret;
1496 
1497 	/*
1498 	 * Wait until kthreadd is all set-up.
1499 	 */
1500 	wait_for_completion(&kthreadd_done);
1501 
1502 	kernel_init_freeable();
1503 	/* need to finish all async __init code before freeing the memory */
1504 	async_synchronize_full();
1505 
1506 	system_state = SYSTEM_FREEING_INITMEM;
1507 	kprobe_free_init_mem();
1508 	ftrace_free_init_mem();
1509 	kgdb_free_init_mem();
1510 	exit_boot_config();
1511 	free_initmem();
1512 	mark_readonly();
1513 
1514 	/*
1515 	 * Kernel mappings are now finalized - update the userspace page-table
1516 	 * to finalize PTI.
1517 	 */
1518 	pti_finalize();
1519 
1520 	system_state = SYSTEM_RUNNING;
1521 	numa_default_policy();
1522 
1523 	rcu_end_inkernel_boot();
1524 
1525 	do_sysctl_args();
1526 
1527 	if (ramdisk_execute_command) {
1528 		ret = run_init_process(ramdisk_execute_command);
1529 		if (!ret)
1530 			return 0;
1531 		pr_err("Failed to execute %s (error %d)\n",
1532 		       ramdisk_execute_command, ret);
1533 	}
1534 
1535 	/*
1536 	 * We try each of these until one succeeds.
1537 	 *
1538 	 * The Bourne shell can be used instead of init if we are
1539 	 * trying to recover a really broken machine.
1540 	 */
1541 	if (execute_command) {
1542 		ret = run_init_process(execute_command);
1543 		if (!ret)
1544 			return 0;
1545 		panic("Requested init %s failed (error %d).",
1546 		      execute_command, ret);
1547 	}
1548 
1549 	if (CONFIG_DEFAULT_INIT[0] != '\0') {
1550 		ret = run_init_process(CONFIG_DEFAULT_INIT);
1551 		if (ret)
1552 			pr_err("Default init %s failed (error %d)\n",
1553 			       CONFIG_DEFAULT_INIT, ret);
1554 		else
1555 			return 0;
1556 	}
1557 
1558 	if (!try_to_run_init_process("/sbin/init") ||
1559 	    !try_to_run_init_process("/etc/init") ||
1560 	    !try_to_run_init_process("/bin/init") ||
1561 	    !try_to_run_init_process("/bin/sh"))
1562 		return 0;
1563 
1564 	panic("No working init found.  Try passing init= option to kernel. "
1565 	      "See Linux Documentation/admin-guide/init.rst for guidance.");
1566 }
1567 
1568 /* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1569 void __init console_on_rootfs(void)
1570 {
1571 	struct file *file = filp_open("/dev/console", O_RDWR, 0);
1572 
1573 	if (IS_ERR(file)) {
1574 		pr_err("Warning: unable to open an initial console.\n");
1575 		return;
1576 	}
1577 	init_dup(file);
1578 	init_dup(file);
1579 	init_dup(file);
1580 	fput(file);
1581 }
1582 
1583 static noinline void __init kernel_init_freeable(void)
1584 {
1585 	/* Now the scheduler is fully set up and can do blocking allocations */
1586 	gfp_allowed_mask = __GFP_BITS_MASK;
1587 
1588 	/*
1589 	 * init can allocate pages on any node
1590 	 */
1591 	set_mems_allowed(node_states[N_MEMORY]);
1592 
1593 	cad_pid = get_pid(task_pid(current));
1594 
1595 	smp_prepare_cpus(setup_max_cpus);
1596 
1597 	workqueue_init();
1598 
1599 	init_mm_internals();
1600 
1601 	rcu_init_tasks_generic();
1602 	do_pre_smp_initcalls();
1603 	lockup_detector_init();
1604 
1605 	smp_init();
1606 	sched_init_smp();
1607 
1608 	padata_init();
1609 	page_alloc_init_late();
1610 	/* Initialize page ext after all struct pages are initialized. */
1611 	page_ext_init();
1612 
1613 	do_basic_setup();
1614 
1615 	kunit_run_all_tests();
1616 
1617 	wait_for_initramfs();
1618 	console_on_rootfs();
1619 
1620 	/*
1621 	 * check if there is an early userspace init.  If yes, let it do all
1622 	 * the work
1623 	 */
1624 	if (init_eaccess(ramdisk_execute_command) != 0) {
1625 		ramdisk_execute_command = NULL;
1626 		prepare_namespace();
1627 	}
1628 
1629 	/*
1630 	 * Ok, we have completed the initial bootup, and
1631 	 * we're essentially up and running. Get rid of the
1632 	 * initmem segments and start the user-mode stuff..
1633 	 *
1634 	 * rootfs is available now, try loading the public keys
1635 	 * and default modules
1636 	 */
1637 
1638 	integrity_load_keys();
1639 }
1640