1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * handle transition of Linux booting another kernel
4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
5 */
6
7 #define pr_fmt(fmt) "kexec: " fmt
8
9 #include <linux/mm.h>
10 #include <linux/kexec.h>
11 #include <linux/string.h>
12 #include <linux/gfp.h>
13 #include <linux/reboot.h>
14 #include <linux/numa.h>
15 #include <linux/ftrace.h>
16 #include <linux/io.h>
17 #include <linux/suspend.h>
18 #include <linux/vmalloc.h>
19 #include <linux/efi.h>
20 #include <linux/cc_platform.h>
21
22 #include <asm/init.h>
23 #include <asm/tlbflush.h>
24 #include <asm/mmu_context.h>
25 #include <asm/io_apic.h>
26 #include <asm/debugreg.h>
27 #include <asm/kexec-bzimage64.h>
28 #include <asm/setup.h>
29 #include <asm/set_memory.h>
30 #include <asm/cpu.h>
31 #include <asm/efi.h>
32
33 #ifdef CONFIG_ACPI
34 /*
35 * Used while adding mapping for ACPI tables.
36 * Can be reused when other iomem regions need be mapped
37 */
38 struct init_pgtable_data {
39 struct x86_mapping_info *info;
40 pgd_t *level4p;
41 };
42
mem_region_callback(struct resource * res,void * arg)43 static int mem_region_callback(struct resource *res, void *arg)
44 {
45 struct init_pgtable_data *data = arg;
46
47 return kernel_ident_mapping_init(data->info, data->level4p,
48 res->start, res->end + 1);
49 }
50
51 static int
map_acpi_tables(struct x86_mapping_info * info,pgd_t * level4p)52 map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p)
53 {
54 struct init_pgtable_data data;
55 unsigned long flags;
56 int ret;
57
58 data.info = info;
59 data.level4p = level4p;
60 flags = IORESOURCE_MEM | IORESOURCE_BUSY;
61
62 ret = walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1,
63 &data, mem_region_callback);
64 if (ret && ret != -EINVAL)
65 return ret;
66
67 /* ACPI tables could be located in ACPI Non-volatile Storage region */
68 ret = walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1,
69 &data, mem_region_callback);
70 if (ret && ret != -EINVAL)
71 return ret;
72
73 return 0;
74 }
75 #else
map_acpi_tables(struct x86_mapping_info * info,pgd_t * level4p)76 static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; }
77 #endif
78
79 #ifdef CONFIG_KEXEC_FILE
80 const struct kexec_file_ops * const kexec_file_loaders[] = {
81 &kexec_bzImage64_ops,
82 NULL
83 };
84 #endif
85
86 static int
map_efi_systab(struct x86_mapping_info * info,pgd_t * level4p)87 map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p)
88 {
89 #ifdef CONFIG_EFI
90 unsigned long mstart, mend;
91 void *kaddr;
92 int ret;
93
94 if (!efi_enabled(EFI_BOOT))
95 return 0;
96
97 mstart = (boot_params.efi_info.efi_systab |
98 ((u64)boot_params.efi_info.efi_systab_hi<<32));
99
100 if (efi_enabled(EFI_64BIT))
101 mend = mstart + sizeof(efi_system_table_64_t);
102 else
103 mend = mstart + sizeof(efi_system_table_32_t);
104
105 if (!mstart)
106 return 0;
107
108 ret = kernel_ident_mapping_init(info, level4p, mstart, mend);
109 if (ret)
110 return ret;
111
112 kaddr = memremap(mstart, mend - mstart, MEMREMAP_WB);
113 if (!kaddr) {
114 pr_err("Could not map UEFI system table\n");
115 return -ENOMEM;
116 }
117
118 mstart = efi_config_table;
119
120 if (efi_enabled(EFI_64BIT)) {
121 efi_system_table_64_t *stbl = (efi_system_table_64_t *)kaddr;
122
123 mend = mstart + sizeof(efi_config_table_64_t) * stbl->nr_tables;
124 } else {
125 efi_system_table_32_t *stbl = (efi_system_table_32_t *)kaddr;
126
127 mend = mstart + sizeof(efi_config_table_32_t) * stbl->nr_tables;
128 }
129
130 memunmap(kaddr);
131
132 return kernel_ident_mapping_init(info, level4p, mstart, mend);
133 #endif
134 return 0;
135 }
136
free_transition_pgtable(struct kimage * image)137 static void free_transition_pgtable(struct kimage *image)
138 {
139 free_page((unsigned long)image->arch.p4d);
140 image->arch.p4d = NULL;
141 free_page((unsigned long)image->arch.pud);
142 image->arch.pud = NULL;
143 free_page((unsigned long)image->arch.pmd);
144 image->arch.pmd = NULL;
145 free_page((unsigned long)image->arch.pte);
146 image->arch.pte = NULL;
147 }
148
init_transition_pgtable(struct kimage * image,pgd_t * pgd)149 static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
150 {
151 pgprot_t prot = PAGE_KERNEL_EXEC_NOENC;
152 unsigned long vaddr, paddr;
153 int result = -ENOMEM;
154 p4d_t *p4d;
155 pud_t *pud;
156 pmd_t *pmd;
157 pte_t *pte;
158
159 vaddr = (unsigned long)relocate_kernel;
160 paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
161 pgd += pgd_index(vaddr);
162 if (!pgd_present(*pgd)) {
163 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
164 if (!p4d)
165 goto err;
166 image->arch.p4d = p4d;
167 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
168 }
169 p4d = p4d_offset(pgd, vaddr);
170 if (!p4d_present(*p4d)) {
171 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
172 if (!pud)
173 goto err;
174 image->arch.pud = pud;
175 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
176 }
177 pud = pud_offset(p4d, vaddr);
178 if (!pud_present(*pud)) {
179 pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
180 if (!pmd)
181 goto err;
182 image->arch.pmd = pmd;
183 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
184 }
185 pmd = pmd_offset(pud, vaddr);
186 if (!pmd_present(*pmd)) {
187 pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
188 if (!pte)
189 goto err;
190 image->arch.pte = pte;
191 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
192 }
193 pte = pte_offset_kernel(pmd, vaddr);
194
195 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
196 prot = PAGE_KERNEL_EXEC;
197
198 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
199 return 0;
200 err:
201 return result;
202 }
203
alloc_pgt_page(void * data)204 static void *alloc_pgt_page(void *data)
205 {
206 struct kimage *image = (struct kimage *)data;
207 struct page *page;
208 void *p = NULL;
209
210 page = kimage_alloc_control_pages(image, 0);
211 if (page) {
212 p = page_address(page);
213 clear_page(p);
214 }
215
216 return p;
217 }
218
init_pgtable(struct kimage * image,unsigned long start_pgtable)219 static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
220 {
221 struct x86_mapping_info info = {
222 .alloc_pgt_page = alloc_pgt_page,
223 .context = image,
224 .page_flag = __PAGE_KERNEL_LARGE_EXEC,
225 .kernpg_flag = _KERNPG_TABLE_NOENC,
226 };
227 unsigned long mstart, mend;
228 pgd_t *level4p;
229 int result;
230 int i;
231
232 level4p = (pgd_t *)__va(start_pgtable);
233 clear_page(level4p);
234
235 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
236 info.page_flag |= _PAGE_ENC;
237 info.kernpg_flag |= _PAGE_ENC;
238 }
239
240 if (direct_gbpages)
241 info.direct_gbpages = true;
242
243 for (i = 0; i < nr_pfn_mapped; i++) {
244 mstart = pfn_mapped[i].start << PAGE_SHIFT;
245 mend = pfn_mapped[i].end << PAGE_SHIFT;
246
247 result = kernel_ident_mapping_init(&info,
248 level4p, mstart, mend);
249 if (result)
250 return result;
251 }
252
253 /*
254 * segments's mem ranges could be outside 0 ~ max_pfn,
255 * for example when jump back to original kernel from kexeced kernel.
256 * or first kernel is booted with user mem map, and second kernel
257 * could be loaded out of that range.
258 */
259 for (i = 0; i < image->nr_segments; i++) {
260 mstart = image->segment[i].mem;
261 mend = mstart + image->segment[i].memsz;
262
263 result = kernel_ident_mapping_init(&info,
264 level4p, mstart, mend);
265
266 if (result)
267 return result;
268 }
269
270 /*
271 * Prepare EFI systab and ACPI tables for kexec kernel since they are
272 * not covered by pfn_mapped.
273 */
274 result = map_efi_systab(&info, level4p);
275 if (result)
276 return result;
277
278 result = map_acpi_tables(&info, level4p);
279 if (result)
280 return result;
281
282 return init_transition_pgtable(image, level4p);
283 }
284
load_segments(void)285 static void load_segments(void)
286 {
287 __asm__ __volatile__ (
288 "\tmovl %0,%%ds\n"
289 "\tmovl %0,%%es\n"
290 "\tmovl %0,%%ss\n"
291 "\tmovl %0,%%fs\n"
292 "\tmovl %0,%%gs\n"
293 : : "a" (__KERNEL_DS) : "memory"
294 );
295 }
296
machine_kexec_prepare(struct kimage * image)297 int machine_kexec_prepare(struct kimage *image)
298 {
299 unsigned long start_pgtable;
300 int result;
301
302 /* Calculate the offsets */
303 start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT;
304
305 /* Setup the identity mapped 64bit page table */
306 result = init_pgtable(image, start_pgtable);
307 if (result)
308 return result;
309
310 return 0;
311 }
312
machine_kexec_cleanup(struct kimage * image)313 void machine_kexec_cleanup(struct kimage *image)
314 {
315 free_transition_pgtable(image);
316 }
317
318 /*
319 * Do not allocate memory (or fail in any way) in machine_kexec().
320 * We are past the point of no return, committed to rebooting now.
321 */
machine_kexec(struct kimage * image)322 void machine_kexec(struct kimage *image)
323 {
324 unsigned long page_list[PAGES_NR];
325 unsigned int host_mem_enc_active;
326 int save_ftrace_enabled;
327 void *control_page;
328
329 /*
330 * This must be done before load_segments() since if call depth tracking
331 * is used then GS must be valid to make any function calls.
332 */
333 host_mem_enc_active = cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT);
334
335 #ifdef CONFIG_KEXEC_JUMP
336 if (image->preserve_context)
337 save_processor_state();
338 #endif
339
340 save_ftrace_enabled = __ftrace_enabled_save();
341
342 /* Interrupts aren't acceptable while we reboot */
343 local_irq_disable();
344 hw_breakpoint_disable();
345 cet_disable();
346
347 if (image->preserve_context) {
348 #ifdef CONFIG_X86_IO_APIC
349 /*
350 * We need to put APICs in legacy mode so that we can
351 * get timer interrupts in second kernel. kexec/kdump
352 * paths already have calls to restore_boot_irq_mode()
353 * in one form or other. kexec jump path also need one.
354 */
355 clear_IO_APIC();
356 restore_boot_irq_mode();
357 #endif
358 }
359
360 control_page = page_address(image->control_code_page) + PAGE_SIZE;
361 __memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
362
363 page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
364 page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
365 page_list[PA_TABLE_PAGE] =
366 (unsigned long)__pa(page_address(image->control_code_page));
367
368 if (image->type == KEXEC_TYPE_DEFAULT)
369 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
370 << PAGE_SHIFT);
371
372 /*
373 * The segment registers are funny things, they have both a
374 * visible and an invisible part. Whenever the visible part is
375 * set to a specific selector, the invisible part is loaded
376 * with from a table in memory. At no other time is the
377 * descriptor table in memory accessed.
378 *
379 * I take advantage of this here by force loading the
380 * segments, before I zap the gdt with an invalid value.
381 */
382 load_segments();
383 /*
384 * The gdt & idt are now invalid.
385 * If you want to load them you must set up your own idt & gdt.
386 */
387 native_idt_invalidate();
388 native_gdt_invalidate();
389
390 /* now call it */
391 image->start = relocate_kernel((unsigned long)image->head,
392 (unsigned long)page_list,
393 image->start,
394 image->preserve_context,
395 host_mem_enc_active);
396
397 #ifdef CONFIG_KEXEC_JUMP
398 if (image->preserve_context)
399 restore_processor_state();
400 #endif
401
402 __ftrace_enabled_restore(save_ftrace_enabled);
403 }
404
405 /* arch-dependent functionality related to kexec file-based syscall */
406
407 #ifdef CONFIG_KEXEC_FILE
408 /*
409 * Apply purgatory relocations.
410 *
411 * @pi: Purgatory to be relocated.
412 * @section: Section relocations applying to.
413 * @relsec: Section containing RELAs.
414 * @symtabsec: Corresponding symtab.
415 *
416 * TODO: Some of the code belongs to generic code. Move that in kexec.c.
417 */
arch_kexec_apply_relocations_add(struct purgatory_info * pi,Elf_Shdr * section,const Elf_Shdr * relsec,const Elf_Shdr * symtabsec)418 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
419 Elf_Shdr *section, const Elf_Shdr *relsec,
420 const Elf_Shdr *symtabsec)
421 {
422 unsigned int i;
423 Elf64_Rela *rel;
424 Elf64_Sym *sym;
425 void *location;
426 unsigned long address, sec_base, value;
427 const char *strtab, *name, *shstrtab;
428 const Elf_Shdr *sechdrs;
429
430 /* String & section header string table */
431 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
432 strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset;
433 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
434
435 rel = (void *)pi->ehdr + relsec->sh_offset;
436
437 pr_debug("Applying relocate section %s to %u\n",
438 shstrtab + relsec->sh_name, relsec->sh_info);
439
440 for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) {
441
442 /*
443 * rel[i].r_offset contains byte offset from beginning
444 * of section to the storage unit affected.
445 *
446 * This is location to update. This is temporary buffer
447 * where section is currently loaded. This will finally be
448 * loaded to a different address later, pointed to by
449 * ->sh_addr. kexec takes care of moving it
450 * (kexec_load_segment()).
451 */
452 location = pi->purgatory_buf;
453 location += section->sh_offset;
454 location += rel[i].r_offset;
455
456 /* Final address of the location */
457 address = section->sh_addr + rel[i].r_offset;
458
459 /*
460 * rel[i].r_info contains information about symbol table index
461 * w.r.t which relocation must be made and type of relocation
462 * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
463 * these respectively.
464 */
465 sym = (void *)pi->ehdr + symtabsec->sh_offset;
466 sym += ELF64_R_SYM(rel[i].r_info);
467
468 if (sym->st_name)
469 name = strtab + sym->st_name;
470 else
471 name = shstrtab + sechdrs[sym->st_shndx].sh_name;
472
473 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
474 name, sym->st_info, sym->st_shndx, sym->st_value,
475 sym->st_size);
476
477 if (sym->st_shndx == SHN_UNDEF) {
478 pr_err("Undefined symbol: %s\n", name);
479 return -ENOEXEC;
480 }
481
482 if (sym->st_shndx == SHN_COMMON) {
483 pr_err("symbol '%s' in common section\n", name);
484 return -ENOEXEC;
485 }
486
487 if (sym->st_shndx == SHN_ABS)
488 sec_base = 0;
489 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
490 pr_err("Invalid section %d for symbol %s\n",
491 sym->st_shndx, name);
492 return -ENOEXEC;
493 } else
494 sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
495
496 value = sym->st_value;
497 value += sec_base;
498 value += rel[i].r_addend;
499
500 switch (ELF64_R_TYPE(rel[i].r_info)) {
501 case R_X86_64_NONE:
502 break;
503 case R_X86_64_64:
504 *(u64 *)location = value;
505 break;
506 case R_X86_64_32:
507 *(u32 *)location = value;
508 if (value != *(u32 *)location)
509 goto overflow;
510 break;
511 case R_X86_64_32S:
512 *(s32 *)location = value;
513 if ((s64)value != *(s32 *)location)
514 goto overflow;
515 break;
516 case R_X86_64_PC32:
517 case R_X86_64_PLT32:
518 value -= (u64)address;
519 *(u32 *)location = value;
520 break;
521 default:
522 pr_err("Unknown rela relocation: %llu\n",
523 ELF64_R_TYPE(rel[i].r_info));
524 return -ENOEXEC;
525 }
526 }
527 return 0;
528
529 overflow:
530 pr_err("Overflow in relocation type %d value 0x%lx\n",
531 (int)ELF64_R_TYPE(rel[i].r_info), value);
532 return -ENOEXEC;
533 }
534
arch_kimage_file_post_load_cleanup(struct kimage * image)535 int arch_kimage_file_post_load_cleanup(struct kimage *image)
536 {
537 vfree(image->elf_headers);
538 image->elf_headers = NULL;
539 image->elf_headers_sz = 0;
540
541 return kexec_image_post_load_cleanup_default(image);
542 }
543 #endif /* CONFIG_KEXEC_FILE */
544
545 #ifdef CONFIG_CRASH_DUMP
546
547 static int
kexec_mark_range(unsigned long start,unsigned long end,bool protect)548 kexec_mark_range(unsigned long start, unsigned long end, bool protect)
549 {
550 struct page *page;
551 unsigned int nr_pages;
552
553 /*
554 * For physical range: [start, end]. We must skip the unassigned
555 * crashk resource with zero-valued "end" member.
556 */
557 if (!end || start > end)
558 return 0;
559
560 page = pfn_to_page(start >> PAGE_SHIFT);
561 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
562 if (protect)
563 return set_pages_ro(page, nr_pages);
564 else
565 return set_pages_rw(page, nr_pages);
566 }
567
kexec_mark_crashkres(bool protect)568 static void kexec_mark_crashkres(bool protect)
569 {
570 unsigned long control;
571
572 kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
573
574 /* Don't touch the control code page used in crash_kexec().*/
575 control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
576 /* Control code page is located in the 2nd page. */
577 kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect);
578 control += KEXEC_CONTROL_PAGE_SIZE;
579 kexec_mark_range(control, crashk_res.end, protect);
580 }
581
arch_kexec_protect_crashkres(void)582 void arch_kexec_protect_crashkres(void)
583 {
584 kexec_mark_crashkres(true);
585 }
586
arch_kexec_unprotect_crashkres(void)587 void arch_kexec_unprotect_crashkres(void)
588 {
589 kexec_mark_crashkres(false);
590 }
591 #endif
592
593 /*
594 * During a traditional boot under SME, SME will encrypt the kernel,
595 * so the SME kexec kernel also needs to be un-encrypted in order to
596 * replicate a normal SME boot.
597 *
598 * During a traditional boot under SEV, the kernel has already been
599 * loaded encrypted, so the SEV kexec kernel needs to be encrypted in
600 * order to replicate a normal SEV boot.
601 */
arch_kexec_post_alloc_pages(void * vaddr,unsigned int pages,gfp_t gfp)602 int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)
603 {
604 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
605 return 0;
606
607 /*
608 * If host memory encryption is active we need to be sure that kexec
609 * pages are not encrypted because when we boot to the new kernel the
610 * pages won't be accessed encrypted (initially).
611 */
612 return set_memory_decrypted((unsigned long)vaddr, pages);
613 }
614
arch_kexec_pre_free_pages(void * vaddr,unsigned int pages)615 void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)
616 {
617 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
618 return;
619
620 /*
621 * If host memory encryption is active we need to reset the pages back
622 * to being an encrypted mapping before freeing them.
623 */
624 set_memory_encrypted((unsigned long)vaddr, pages);
625 }
626