1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * handle transition of Linux booting another kernel 4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7 #include <linux/mm.h> 8 #include <linux/kexec.h> 9 #include <linux/delay.h> 10 #include <linux/numa.h> 11 #include <linux/ftrace.h> 12 #include <linux/suspend.h> 13 #include <linux/gfp.h> 14 #include <linux/io.h> 15 16 #include <asm/pgalloc.h> 17 #include <asm/tlbflush.h> 18 #include <asm/mmu_context.h> 19 #include <asm/apic.h> 20 #include <asm/io_apic.h> 21 #include <asm/cpufeature.h> 22 #include <asm/desc.h> 23 #include <asm/set_memory.h> 24 #include <asm/debugreg.h> 25 26 static void load_segments(void) 27 { 28 #define __STR(X) #X 29 #define STR(X) __STR(X) 30 31 __asm__ __volatile__ ( 32 "\tljmp $"STR(__KERNEL_CS)",$1f\n" 33 "\t1:\n" 34 "\tmovl $"STR(__KERNEL_DS)",%%eax\n" 35 "\tmovl %%eax,%%ds\n" 36 "\tmovl %%eax,%%es\n" 37 "\tmovl %%eax,%%ss\n" 38 : : : "eax", "memory"); 39 #undef STR 40 #undef __STR 41 } 42 43 static void machine_kexec_free_page_tables(struct kimage *image) 44 { 45 free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER); 46 image->arch.pgd = NULL; 47 #ifdef CONFIG_X86_PAE 48 free_page((unsigned long)image->arch.pmd0); 49 image->arch.pmd0 = NULL; 50 free_page((unsigned long)image->arch.pmd1); 51 image->arch.pmd1 = NULL; 52 #endif 53 free_page((unsigned long)image->arch.pte0); 54 image->arch.pte0 = NULL; 55 free_page((unsigned long)image->arch.pte1); 56 image->arch.pte1 = NULL; 57 } 58 59 static int machine_kexec_alloc_page_tables(struct kimage *image) 60 { 61 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 62 PGD_ALLOCATION_ORDER); 63 #ifdef CONFIG_X86_PAE 64 image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL); 65 image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL); 66 #endif 67 image->arch.pte0 = (pte_t *)get_zeroed_page(GFP_KERNEL); 68 image->arch.pte1 = (pte_t *)get_zeroed_page(GFP_KERNEL); 69 if (!image->arch.pgd || 70 #ifdef CONFIG_X86_PAE 71 !image->arch.pmd0 || !image->arch.pmd1 || 72 #endif 73 !image->arch.pte0 || !image->arch.pte1) { 74 return -ENOMEM; 75 } 76 return 0; 77 } 78 79 static void machine_kexec_page_table_set_one( 80 pgd_t *pgd, pmd_t *pmd, pte_t *pte, 81 unsigned long vaddr, unsigned long paddr) 82 { 83 p4d_t *p4d; 84 pud_t *pud; 85 86 pgd += pgd_index(vaddr); 87 #ifdef CONFIG_X86_PAE 88 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) 89 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); 90 #endif 91 p4d = p4d_offset(pgd, vaddr); 92 pud = pud_offset(p4d, vaddr); 93 pmd = pmd_offset(pud, vaddr); 94 if (!(pmd_val(*pmd) & _PAGE_PRESENT)) 95 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 96 pte = pte_offset_kernel(pmd, vaddr); 97 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC)); 98 } 99 100 static void machine_kexec_prepare_page_tables(struct kimage *image) 101 { 102 void *control_page; 103 pmd_t *pmd = NULL; 104 105 control_page = page_address(image->control_code_page); 106 #ifdef CONFIG_X86_PAE 107 pmd = image->arch.pmd0; 108 #endif 109 machine_kexec_page_table_set_one( 110 image->arch.pgd, pmd, image->arch.pte0, 111 (unsigned long)control_page, __pa(control_page)); 112 #ifdef CONFIG_X86_PAE 113 pmd = image->arch.pmd1; 114 #endif 115 machine_kexec_page_table_set_one( 116 image->arch.pgd, pmd, image->arch.pte1, 117 __pa(control_page), __pa(control_page)); 118 } 119 120 /* 121 * A architecture hook called to validate the 122 * proposed image and prepare the control pages 123 * as needed. The pages for KEXEC_CONTROL_PAGE_SIZE 124 * have been allocated, but the segments have yet 125 * been copied into the kernel. 126 * 127 * Do what every setup is needed on image and the 128 * reboot code buffer to allow us to avoid allocations 129 * later. 130 * 131 * - Make control page executable. 132 * - Allocate page tables 133 * - Setup page tables 134 */ 135 int machine_kexec_prepare(struct kimage *image) 136 { 137 int error; 138 139 set_memory_x((unsigned long)page_address(image->control_code_page), 1); 140 error = machine_kexec_alloc_page_tables(image); 141 if (error) 142 return error; 143 machine_kexec_prepare_page_tables(image); 144 return 0; 145 } 146 147 /* 148 * Undo anything leftover by machine_kexec_prepare 149 * when an image is freed. 150 */ 151 void machine_kexec_cleanup(struct kimage *image) 152 { 153 set_memory_nx((unsigned long)page_address(image->control_code_page), 1); 154 machine_kexec_free_page_tables(image); 155 } 156 157 /* 158 * Do not allocate memory (or fail in any way) in machine_kexec(). 159 * We are past the point of no return, committed to rebooting now. 160 */ 161 void machine_kexec(struct kimage *image) 162 { 163 relocate_kernel_fn *relocate_kernel_ptr; 164 unsigned long page_list[PAGES_NR]; 165 void *control_page; 166 int save_ftrace_enabled; 167 168 #ifdef CONFIG_KEXEC_JUMP 169 if (image->preserve_context) 170 save_processor_state(); 171 #endif 172 173 save_ftrace_enabled = __ftrace_enabled_save(); 174 175 /* Interrupts aren't acceptable while we reboot */ 176 local_irq_disable(); 177 hw_breakpoint_disable(); 178 179 if (image->preserve_context) { 180 #ifdef CONFIG_X86_IO_APIC 181 /* 182 * We need to put APICs in legacy mode so that we can 183 * get timer interrupts in second kernel. kexec/kdump 184 * paths already have calls to restore_boot_irq_mode() 185 * in one form or other. kexec jump path also need one. 186 */ 187 clear_IO_APIC(); 188 restore_boot_irq_mode(); 189 #endif 190 } 191 192 control_page = page_address(image->control_code_page); 193 memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE); 194 195 relocate_kernel_ptr = control_page; 196 page_list[PA_CONTROL_PAGE] = __pa(control_page); 197 page_list[VA_CONTROL_PAGE] = (unsigned long)control_page; 198 page_list[PA_PGD] = __pa(image->arch.pgd); 199 200 if (image->type == KEXEC_TYPE_DEFAULT) 201 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page) 202 << PAGE_SHIFT); 203 204 /* 205 * The segment registers are funny things, they have both a 206 * visible and an invisible part. Whenever the visible part is 207 * set to a specific selector, the invisible part is loaded 208 * with from a table in memory. At no other time is the 209 * descriptor table in memory accessed. 210 * 211 * I take advantage of this here by force loading the 212 * segments, before I zap the gdt with an invalid value. 213 */ 214 load_segments(); 215 /* 216 * The gdt & idt are now invalid. 217 * If you want to load them you must set up your own idt & gdt. 218 */ 219 native_idt_invalidate(); 220 native_gdt_invalidate(); 221 222 /* now call it */ 223 image->start = relocate_kernel_ptr((unsigned long)image->head, 224 (unsigned long)page_list, 225 image->start, 226 boot_cpu_has(X86_FEATURE_PAE), 227 image->preserve_context); 228 229 #ifdef CONFIG_KEXEC_JUMP 230 if (image->preserve_context) 231 restore_processor_state(); 232 #endif 233 234 __ftrace_enabled_restore(save_ftrace_enabled); 235 } 236