1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * machine_kexec.c for kexec
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7 #include <linux/compiler.h>
8 #include <linux/cpu.h>
9 #include <linux/kexec.h>
10 #include <linux/crash_dump.h>
11 #include <linux/delay.h>
12 #include <linux/irq.h>
13 #include <linux/libfdt.h>
14 #include <linux/mm.h>
15 #include <linux/of_fdt.h>
16 #include <linux/reboot.h>
17 #include <linux/sched.h>
18 #include <linux/sched/task_stack.h>
19
20 #include <asm/bootinfo.h>
21 #include <asm/cacheflush.h>
22 #include <asm/page.h>
23
24 /* 0x100000 ~ 0x200000 is safe */
25 #define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL)
26 #define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL)
27
28 static unsigned long reboot_code_buffer;
29 static cpumask_t cpus_in_crash = CPU_MASK_NONE;
30
31 #ifdef CONFIG_SMP
32 static void (*relocated_kexec_smp_wait)(void *);
33 atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0);
34 #endif
35
36 static unsigned long efi_boot;
37 static unsigned long cmdline_ptr;
38 static unsigned long systable_ptr;
39 static unsigned long start_addr;
40 static unsigned long first_ind_entry;
41
machine_kexec_prepare(struct kimage * kimage)42 int machine_kexec_prepare(struct kimage *kimage)
43 {
44 int i;
45 char head[8];
46 char *bootloader = "kexec";
47 void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR;
48
49 kimage->arch.efi_boot = fw_arg0;
50 kimage->arch.systable_ptr = fw_arg2;
51
52 if (kimage->file_mode == 1) {
53 /*
54 * kimage->cmdline_buf will be released in kexec_file_load, so copy
55 * to the KEXEC_CMDLINE_ADDR safe area.
56 */
57 memcpy((void *)KEXEC_CMDLINE_ADDR, (void *)kimage->arch.cmdline_ptr,
58 strlen((char *)kimage->arch.cmdline_ptr) + 1);
59 kimage->arch.cmdline_ptr = (unsigned long)KEXEC_CMDLINE_ADDR;
60 } else {
61 /* Find the command line */
62 for (i = 0; i < kimage->nr_segments; i++) {
63 if (copy_from_user(head, kimage->segment[i].buf, strlen(bootloader)))
64 continue;
65 if (!strncmp(bootloader, head, strlen(bootloader))) {
66 if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE))
67 kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr;
68 break;
69 }
70 }
71
72 if (!kimage->arch.cmdline_ptr) {
73 pr_err("Command line not included in the provided image\n");
74 return -EINVAL;
75 }
76 }
77
78 /* kexec/kdump need a safe page to save reboot_code_buffer */
79 kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE);
80
81 reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page);
82 memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size);
83
84 #ifdef CONFIG_SMP
85 /* All secondary cpus now may jump to kexec_smp_wait cycle */
86 relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel);
87 #endif
88
89 return 0;
90 }
91
machine_kexec_cleanup(struct kimage * kimage)92 void machine_kexec_cleanup(struct kimage *kimage)
93 {
94 }
95
kexec_reboot(void)96 void kexec_reboot(void)
97 {
98 do_kexec_t do_kexec = NULL;
99
100 /*
101 * We know we were online, and there will be no incoming IPIs at
102 * this point. Mark online again before rebooting so that the crash
103 * analysis tool will see us correctly.
104 */
105 set_cpu_online(smp_processor_id(), true);
106
107 /* Ensure remote CPUs observe that we're online before rebooting. */
108 smp_mb__after_atomic();
109
110 /*
111 * Make sure we get correct instructions written by the
112 * machine_kexec_prepare() CPU.
113 */
114 __asm__ __volatile__ ("\tibar 0\n"::);
115
116 #ifdef CONFIG_SMP
117 /* All secondary cpus go to kexec_smp_wait */
118 if (smp_processor_id() > 0) {
119 relocated_kexec_smp_wait(NULL);
120 BUG();
121 }
122 #endif
123
124 do_kexec = (void *)reboot_code_buffer;
125 do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry);
126
127 BUG();
128 }
129
130
131 #ifdef CONFIG_SMP
kexec_shutdown_secondary(void * regs)132 static void kexec_shutdown_secondary(void *regs)
133 {
134 int cpu = smp_processor_id();
135
136 if (!cpu_online(cpu))
137 return;
138
139 /* We won't be sent IPIs any more. */
140 set_cpu_online(cpu, false);
141
142 local_irq_disable();
143 while (!atomic_read(&kexec_ready_to_reboot))
144 cpu_relax();
145
146 kexec_reboot();
147 }
148
crash_shutdown_secondary(void * passed_regs)149 static void crash_shutdown_secondary(void *passed_regs)
150 {
151 int cpu = smp_processor_id();
152 struct pt_regs *regs = passed_regs;
153
154 /*
155 * If we are passed registers, use those. Otherwise get the
156 * regs from the last interrupt, which should be correct, as
157 * we are in an interrupt. But if the regs are not there,
158 * pull them from the top of the stack. They are probably
159 * wrong, but we need something to keep from crashing again.
160 */
161 if (!regs)
162 regs = get_irq_regs();
163 if (!regs)
164 regs = task_pt_regs(current);
165
166 if (!cpu_online(cpu))
167 return;
168
169 /* We won't be sent IPIs any more. */
170 set_cpu_online(cpu, false);
171
172 local_irq_disable();
173 if (!cpumask_test_cpu(cpu, &cpus_in_crash))
174 crash_save_cpu(regs, cpu);
175 cpumask_set_cpu(cpu, &cpus_in_crash);
176
177 while (!atomic_read(&kexec_ready_to_reboot))
178 cpu_relax();
179
180 kexec_reboot();
181 }
182
crash_smp_send_stop(void)183 void crash_smp_send_stop(void)
184 {
185 unsigned int ncpus;
186 unsigned long timeout;
187 static int cpus_stopped;
188
189 /*
190 * This function can be called twice in panic path, but obviously
191 * we should execute this only once.
192 */
193 if (cpus_stopped)
194 return;
195
196 cpus_stopped = 1;
197
198 /* Excluding the panic cpu */
199 ncpus = num_online_cpus() - 1;
200
201 smp_call_function(crash_shutdown_secondary, NULL, 0);
202 smp_wmb();
203
204 /*
205 * The crash CPU sends an IPI and wait for other CPUs to
206 * respond. Delay of at least 10 seconds.
207 */
208 timeout = MSEC_PER_SEC * 10;
209 pr_emerg("Sending IPI to other cpus...\n");
210 while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) {
211 mdelay(1);
212 cpu_relax();
213 }
214 }
215 #endif /* defined(CONFIG_SMP) */
216
machine_shutdown(void)217 void machine_shutdown(void)
218 {
219 #ifdef CONFIG_SMP
220 int cpu;
221
222 /* All CPUs go to reboot_code_buffer */
223 for_each_possible_cpu(cpu)
224 if (!cpu_online(cpu))
225 cpu_device_up(get_cpu_device(cpu));
226
227 smp_call_function(kexec_shutdown_secondary, NULL, 0);
228 #endif
229 }
230
machine_crash_shutdown(struct pt_regs * regs)231 void machine_crash_shutdown(struct pt_regs *regs)
232 {
233 int crashing_cpu;
234
235 local_irq_disable();
236
237 crashing_cpu = smp_processor_id();
238 crash_save_cpu(regs, crashing_cpu);
239
240 #ifdef CONFIG_SMP
241 crash_smp_send_stop();
242 #endif
243 machine_kexec_mask_interrupts();
244 cpumask_set_cpu(crashing_cpu, &cpus_in_crash);
245
246 pr_info("Starting crashdump kernel...\n");
247 }
248
machine_kexec(struct kimage * image)249 void machine_kexec(struct kimage *image)
250 {
251 unsigned long entry, *ptr;
252 struct kimage_arch *internal = &image->arch;
253
254 efi_boot = internal->efi_boot;
255 cmdline_ptr = internal->cmdline_ptr;
256 systable_ptr = internal->systable_ptr;
257
258 start_addr = (unsigned long)phys_to_virt(image->start);
259
260 first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ?
261 (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0;
262
263 /*
264 * The generic kexec code builds a page list with physical
265 * addresses. they are directly accessible through XKPRANGE
266 * hence the phys_to_virt() call.
267 */
268 for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE);
269 ptr = (entry & IND_INDIRECTION) ?
270 phys_to_virt(entry & PAGE_MASK) : ptr + 1) {
271 if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION ||
272 *ptr & IND_DESTINATION)
273 *ptr = (unsigned long) phys_to_virt(*ptr);
274 }
275
276 /* Mark offline before disabling local irq. */
277 set_cpu_online(smp_processor_id(), false);
278
279 /* We do not want to be bothered. */
280 local_irq_disable();
281 machine_kexec_mask_interrupts();
282
283 pr_notice("EFI boot flag: 0x%lx\n", efi_boot);
284 pr_notice("Command line addr: 0x%lx\n", cmdline_ptr);
285 pr_notice("Command line string: %s\n", (char *)cmdline_ptr);
286 pr_notice("System table addr: 0x%lx\n", systable_ptr);
287 pr_notice("We will call new kernel at 0x%lx\n", start_addr);
288 pr_notice("Bye ...\n");
289
290 /* Make reboot code buffer available to the boot CPU. */
291 flush_cache_all();
292
293 #ifdef CONFIG_SMP
294 atomic_set(&kexec_ready_to_reboot, 1);
295 #endif
296
297 kexec_reboot();
298 }
299