1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * machine_kexec.c for kexec 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/compiler.h> 8 #include <linux/cpu.h> 9 #include <linux/kexec.h> 10 #include <linux/crash_dump.h> 11 #include <linux/delay.h> 12 #include <linux/irq.h> 13 #include <linux/libfdt.h> 14 #include <linux/mm.h> 15 #include <linux/of_fdt.h> 16 #include <linux/reboot.h> 17 #include <linux/sched.h> 18 #include <linux/sched/task_stack.h> 19 20 #include <asm/bootinfo.h> 21 #include <asm/cacheflush.h> 22 #include <asm/page.h> 23 24 /* 0x100000 ~ 0x200000 is safe */ 25 #define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL) 26 #define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL) 27 28 static unsigned long reboot_code_buffer; 29 static cpumask_t cpus_in_crash = CPU_MASK_NONE; 30 31 #ifdef CONFIG_SMP 32 static void (*relocated_kexec_smp_wait)(void *); 33 atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); 34 #endif 35 36 static unsigned long efi_boot; 37 static unsigned long cmdline_ptr; 38 static unsigned long systable_ptr; 39 static unsigned long start_addr; 40 static unsigned long first_ind_entry; 41 42 int machine_kexec_prepare(struct kimage *kimage) 43 { 44 int i; 45 char head[8]; 46 char *bootloader = "kexec"; 47 void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; 48 49 kimage->arch.efi_boot = fw_arg0; 50 kimage->arch.systable_ptr = fw_arg2; 51 52 if (kimage->file_mode == 1) { 53 /* 54 * kimage->cmdline_buf will be released in kexec_file_load, so copy 55 * to the KEXEC_CMDLINE_ADDR safe area. 56 */ 57 memcpy((void *)KEXEC_CMDLINE_ADDR, (void *)kimage->arch.cmdline_ptr, 58 strlen((char *)kimage->arch.cmdline_ptr) + 1); 59 kimage->arch.cmdline_ptr = (unsigned long)KEXEC_CMDLINE_ADDR; 60 } else { 61 /* Find the command line */ 62 for (i = 0; i < kimage->nr_segments; i++) { 63 if (copy_from_user(head, kimage->segment[i].buf, strlen(bootloader))) 64 continue; 65 if (!strncmp(bootloader, head, strlen(bootloader))) { 66 if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) 67 kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; 68 break; 69 } 70 } 71 72 if (!kimage->arch.cmdline_ptr) { 73 pr_err("Command line not included in the provided image\n"); 74 return -EINVAL; 75 } 76 } 77 78 /* kexec/kdump need a safe page to save reboot_code_buffer */ 79 kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE); 80 81 reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page); 82 memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); 83 84 #ifdef CONFIG_SMP 85 /* All secondary cpus now may jump to kexec_smp_wait cycle */ 86 relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel); 87 #endif 88 89 return 0; 90 } 91 92 void machine_kexec_cleanup(struct kimage *kimage) 93 { 94 } 95 96 void kexec_reboot(void) 97 { 98 do_kexec_t do_kexec = NULL; 99 100 /* 101 * We know we were online, and there will be no incoming IPIs at 102 * this point. Mark online again before rebooting so that the crash 103 * analysis tool will see us correctly. 104 */ 105 set_cpu_online(smp_processor_id(), true); 106 107 /* Ensure remote CPUs observe that we're online before rebooting. */ 108 smp_mb__after_atomic(); 109 110 /* 111 * Make sure we get correct instructions written by the 112 * machine_kexec_prepare() CPU. 113 */ 114 __asm__ __volatile__ ("\tibar 0\n"::); 115 116 #ifdef CONFIG_SMP 117 /* All secondary cpus go to kexec_smp_wait */ 118 if (smp_processor_id() > 0) { 119 relocated_kexec_smp_wait(NULL); 120 BUG(); 121 } 122 #endif 123 124 do_kexec = (void *)reboot_code_buffer; 125 do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry); 126 127 BUG(); 128 } 129 130 131 #ifdef CONFIG_SMP 132 static void kexec_shutdown_secondary(void *regs) 133 { 134 int cpu = smp_processor_id(); 135 136 if (!cpu_online(cpu)) 137 return; 138 139 /* We won't be sent IPIs any more. */ 140 set_cpu_online(cpu, false); 141 142 local_irq_disable(); 143 while (!atomic_read(&kexec_ready_to_reboot)) 144 cpu_relax(); 145 146 kexec_reboot(); 147 } 148 149 static void crash_shutdown_secondary(void *passed_regs) 150 { 151 int cpu = smp_processor_id(); 152 struct pt_regs *regs = passed_regs; 153 154 /* 155 * If we are passed registers, use those. Otherwise get the 156 * regs from the last interrupt, which should be correct, as 157 * we are in an interrupt. But if the regs are not there, 158 * pull them from the top of the stack. They are probably 159 * wrong, but we need something to keep from crashing again. 160 */ 161 if (!regs) 162 regs = get_irq_regs(); 163 if (!regs) 164 regs = task_pt_regs(current); 165 166 if (!cpu_online(cpu)) 167 return; 168 169 /* We won't be sent IPIs any more. */ 170 set_cpu_online(cpu, false); 171 172 local_irq_disable(); 173 if (!cpumask_test_cpu(cpu, &cpus_in_crash)) 174 crash_save_cpu(regs, cpu); 175 cpumask_set_cpu(cpu, &cpus_in_crash); 176 177 while (!atomic_read(&kexec_ready_to_reboot)) 178 cpu_relax(); 179 180 kexec_reboot(); 181 } 182 183 void crash_smp_send_stop(void) 184 { 185 unsigned int ncpus; 186 unsigned long timeout; 187 static int cpus_stopped; 188 189 /* 190 * This function can be called twice in panic path, but obviously 191 * we should execute this only once. 192 */ 193 if (cpus_stopped) 194 return; 195 196 cpus_stopped = 1; 197 198 /* Excluding the panic cpu */ 199 ncpus = num_online_cpus() - 1; 200 201 smp_call_function(crash_shutdown_secondary, NULL, 0); 202 smp_wmb(); 203 204 /* 205 * The crash CPU sends an IPI and wait for other CPUs to 206 * respond. Delay of at least 10 seconds. 207 */ 208 timeout = MSEC_PER_SEC * 10; 209 pr_emerg("Sending IPI to other cpus...\n"); 210 while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) { 211 mdelay(1); 212 cpu_relax(); 213 } 214 } 215 #endif /* defined(CONFIG_SMP) */ 216 217 void machine_shutdown(void) 218 { 219 #ifdef CONFIG_SMP 220 int cpu; 221 222 /* All CPUs go to reboot_code_buffer */ 223 for_each_possible_cpu(cpu) 224 if (!cpu_online(cpu)) 225 cpu_device_up(get_cpu_device(cpu)); 226 227 smp_call_function(kexec_shutdown_secondary, NULL, 0); 228 #endif 229 } 230 231 void machine_crash_shutdown(struct pt_regs *regs) 232 { 233 int crashing_cpu; 234 235 local_irq_disable(); 236 237 crashing_cpu = smp_processor_id(); 238 crash_save_cpu(regs, crashing_cpu); 239 240 #ifdef CONFIG_SMP 241 crash_smp_send_stop(); 242 #endif 243 machine_kexec_mask_interrupts(); 244 cpumask_set_cpu(crashing_cpu, &cpus_in_crash); 245 246 pr_info("Starting crashdump kernel...\n"); 247 } 248 249 void machine_kexec(struct kimage *image) 250 { 251 unsigned long entry, *ptr; 252 struct kimage_arch *internal = &image->arch; 253 254 efi_boot = internal->efi_boot; 255 cmdline_ptr = internal->cmdline_ptr; 256 systable_ptr = internal->systable_ptr; 257 258 start_addr = (unsigned long)phys_to_virt(image->start); 259 260 first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ? 261 (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0; 262 263 /* 264 * The generic kexec code builds a page list with physical 265 * addresses. they are directly accessible through XKPRANGE 266 * hence the phys_to_virt() call. 267 */ 268 for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); 269 ptr = (entry & IND_INDIRECTION) ? 270 phys_to_virt(entry & PAGE_MASK) : ptr + 1) { 271 if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION || 272 *ptr & IND_DESTINATION) 273 *ptr = (unsigned long) phys_to_virt(*ptr); 274 } 275 276 /* Mark offline before disabling local irq. */ 277 set_cpu_online(smp_processor_id(), false); 278 279 /* We do not want to be bothered. */ 280 local_irq_disable(); 281 machine_kexec_mask_interrupts(); 282 283 pr_notice("EFI boot flag: 0x%lx\n", efi_boot); 284 pr_notice("Command line addr: 0x%lx\n", cmdline_ptr); 285 pr_notice("Command line string: %s\n", (char *)cmdline_ptr); 286 pr_notice("System table addr: 0x%lx\n", systable_ptr); 287 pr_notice("We will call new kernel at 0x%lx\n", start_addr); 288 pr_notice("Bye ...\n"); 289 290 /* Make reboot code buffer available to the boot CPU. */ 291 flush_cache_all(); 292 293 #ifdef CONFIG_SMP 294 atomic_set(&kexec_ready_to_reboot, 1); 295 #endif 296 297 kexec_reboot(); 298 } 299