1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * handle transition of Linux booting another kernel
4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
5 */
6
7 #define pr_fmt(fmt) "kexec: " fmt
8
9 #include <linux/mm.h>
10 #include <linux/kexec.h>
11 #include <linux/string.h>
12 #include <linux/gfp.h>
13 #include <linux/reboot.h>
14 #include <linux/numa.h>
15 #include <linux/ftrace.h>
16 #include <linux/io.h>
17 #include <linux/suspend.h>
18 #include <linux/vmalloc.h>
19 #include <linux/efi.h>
20 #include <linux/cc_platform.h>
21
22 #include <asm/init.h>
23 #include <asm/tlbflush.h>
24 #include <asm/mmu_context.h>
25 #include <asm/io_apic.h>
26 #include <asm/debugreg.h>
27 #include <asm/kexec-bzimage64.h>
28 #include <asm/setup.h>
29 #include <asm/set_memory.h>
30 #include <asm/cpu.h>
31 #include <asm/efi.h>
32 #include <asm/processor.h>
33
34 #ifdef CONFIG_ACPI
35 /*
36 * Used while adding mapping for ACPI tables.
37 * Can be reused when other iomem regions need be mapped
38 */
39 struct init_pgtable_data {
40 struct x86_mapping_info *info;
41 pgd_t *level4p;
42 };
43
mem_region_callback(struct resource * res,void * arg)44 static int mem_region_callback(struct resource *res, void *arg)
45 {
46 struct init_pgtable_data *data = arg;
47
48 return kernel_ident_mapping_init(data->info, data->level4p,
49 res->start, res->end + 1);
50 }
51
52 static int
map_acpi_tables(struct x86_mapping_info * info,pgd_t * level4p)53 map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p)
54 {
55 struct init_pgtable_data data;
56 unsigned long flags;
57 int ret;
58
59 data.info = info;
60 data.level4p = level4p;
61 flags = IORESOURCE_MEM | IORESOURCE_BUSY;
62
63 ret = walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1,
64 &data, mem_region_callback);
65 if (ret && ret != -EINVAL)
66 return ret;
67
68 /* ACPI tables could be located in ACPI Non-volatile Storage region */
69 ret = walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1,
70 &data, mem_region_callback);
71 if (ret && ret != -EINVAL)
72 return ret;
73
74 return 0;
75 }
76 #else
map_acpi_tables(struct x86_mapping_info * info,pgd_t * level4p)77 static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; }
78 #endif
79
map_mmio_serial(struct x86_mapping_info * info,pgd_t * level4p)80 static int map_mmio_serial(struct x86_mapping_info *info, pgd_t *level4p)
81 {
82 unsigned long mstart, mend;
83
84 if (!kexec_debug_8250_mmio32)
85 return 0;
86
87 mstart = kexec_debug_8250_mmio32 & PAGE_MASK;
88 mend = (kexec_debug_8250_mmio32 + PAGE_SIZE + 23) & PAGE_MASK;
89 pr_info("Map PCI serial at %lx - %lx\n", mstart, mend);
90 return kernel_ident_mapping_init(info, level4p, mstart, mend);
91 }
92
93 #ifdef CONFIG_KEXEC_FILE
94 const struct kexec_file_ops * const kexec_file_loaders[] = {
95 &kexec_bzImage64_ops,
96 NULL
97 };
98 #endif
99
100 static int
map_efi_systab(struct x86_mapping_info * info,pgd_t * level4p)101 map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p)
102 {
103 #ifdef CONFIG_EFI
104 unsigned long mstart, mend;
105 void *kaddr;
106 int ret;
107
108 if (!efi_enabled(EFI_BOOT))
109 return 0;
110
111 mstart = (boot_params.efi_info.efi_systab |
112 ((u64)boot_params.efi_info.efi_systab_hi<<32));
113
114 if (efi_enabled(EFI_64BIT))
115 mend = mstart + sizeof(efi_system_table_64_t);
116 else
117 mend = mstart + sizeof(efi_system_table_32_t);
118
119 if (!mstart)
120 return 0;
121
122 ret = kernel_ident_mapping_init(info, level4p, mstart, mend);
123 if (ret)
124 return ret;
125
126 kaddr = memremap(mstart, mend - mstart, MEMREMAP_WB);
127 if (!kaddr) {
128 pr_err("Could not map UEFI system table\n");
129 return -ENOMEM;
130 }
131
132 mstart = efi_config_table;
133
134 if (efi_enabled(EFI_64BIT)) {
135 efi_system_table_64_t *stbl = (efi_system_table_64_t *)kaddr;
136
137 mend = mstart + sizeof(efi_config_table_64_t) * stbl->nr_tables;
138 } else {
139 efi_system_table_32_t *stbl = (efi_system_table_32_t *)kaddr;
140
141 mend = mstart + sizeof(efi_config_table_32_t) * stbl->nr_tables;
142 }
143
144 memunmap(kaddr);
145
146 return kernel_ident_mapping_init(info, level4p, mstart, mend);
147 #endif
148 return 0;
149 }
150
free_transition_pgtable(struct kimage * image)151 static void free_transition_pgtable(struct kimage *image)
152 {
153 free_page((unsigned long)image->arch.p4d);
154 image->arch.p4d = NULL;
155 free_page((unsigned long)image->arch.pud);
156 image->arch.pud = NULL;
157 free_page((unsigned long)image->arch.pmd);
158 image->arch.pmd = NULL;
159 free_page((unsigned long)image->arch.pte);
160 image->arch.pte = NULL;
161 }
162
init_transition_pgtable(struct kimage * image,pgd_t * pgd,unsigned long control_page)163 static int init_transition_pgtable(struct kimage *image, pgd_t *pgd,
164 unsigned long control_page)
165 {
166 pgprot_t prot = PAGE_KERNEL_EXEC_NOENC;
167 unsigned long vaddr, paddr;
168 int result = -ENOMEM;
169 p4d_t *p4d;
170 pud_t *pud;
171 pmd_t *pmd;
172 pte_t *pte;
173
174 /*
175 * For the transition to the identity mapped page tables, the control
176 * code page also needs to be mapped at the virtual address it starts
177 * off running from.
178 */
179 vaddr = (unsigned long)__va(control_page);
180 paddr = control_page;
181 pgd += pgd_index(vaddr);
182 if (!pgd_present(*pgd)) {
183 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
184 if (!p4d)
185 goto err;
186 image->arch.p4d = p4d;
187 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
188 }
189 p4d = p4d_offset(pgd, vaddr);
190 if (!p4d_present(*p4d)) {
191 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
192 if (!pud)
193 goto err;
194 image->arch.pud = pud;
195 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
196 }
197 pud = pud_offset(p4d, vaddr);
198 if (!pud_present(*pud)) {
199 pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
200 if (!pmd)
201 goto err;
202 image->arch.pmd = pmd;
203 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
204 }
205 pmd = pmd_offset(pud, vaddr);
206 if (!pmd_present(*pmd)) {
207 pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
208 if (!pte)
209 goto err;
210 image->arch.pte = pte;
211 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
212 }
213 pte = pte_offset_kernel(pmd, vaddr);
214
215 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
216 prot = PAGE_KERNEL_EXEC;
217
218 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
219 return 0;
220 err:
221 return result;
222 }
223
alloc_pgt_page(void * data)224 static void *alloc_pgt_page(void *data)
225 {
226 struct kimage *image = (struct kimage *)data;
227 struct page *page;
228 void *p = NULL;
229
230 page = kimage_alloc_control_pages(image, 0);
231 if (page) {
232 p = page_address(page);
233 clear_page(p);
234 }
235
236 return p;
237 }
238
init_pgtable(struct kimage * image,unsigned long control_page)239 static int init_pgtable(struct kimage *image, unsigned long control_page)
240 {
241 struct x86_mapping_info info = {
242 .alloc_pgt_page = alloc_pgt_page,
243 .context = image,
244 .page_flag = __PAGE_KERNEL_LARGE_EXEC,
245 .kernpg_flag = _KERNPG_TABLE_NOENC,
246 };
247 unsigned long mstart, mend;
248 int result;
249 int i;
250
251 image->arch.pgd = alloc_pgt_page(image);
252 if (!image->arch.pgd)
253 return -ENOMEM;
254
255 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
256 info.page_flag |= _PAGE_ENC;
257 info.kernpg_flag |= _PAGE_ENC;
258 }
259
260 if (direct_gbpages)
261 info.direct_gbpages = true;
262
263 for (i = 0; i < nr_pfn_mapped; i++) {
264 mstart = pfn_mapped[i].start << PAGE_SHIFT;
265 mend = pfn_mapped[i].end << PAGE_SHIFT;
266
267 result = kernel_ident_mapping_init(&info, image->arch.pgd,
268 mstart, mend);
269 if (result)
270 return result;
271 }
272
273 /*
274 * segments's mem ranges could be outside 0 ~ max_pfn,
275 * for example when jump back to original kernel from kexeced kernel.
276 * or first kernel is booted with user mem map, and second kernel
277 * could be loaded out of that range.
278 */
279 for (i = 0; i < image->nr_segments; i++) {
280 mstart = image->segment[i].mem;
281 mend = mstart + image->segment[i].memsz;
282
283 result = kernel_ident_mapping_init(&info, image->arch.pgd,
284 mstart, mend);
285
286 if (result)
287 return result;
288 }
289
290 /*
291 * Prepare EFI systab and ACPI tables for kexec kernel since they are
292 * not covered by pfn_mapped.
293 */
294 result = map_efi_systab(&info, image->arch.pgd);
295 if (result)
296 return result;
297
298 result = map_acpi_tables(&info, image->arch.pgd);
299 if (result)
300 return result;
301
302 result = map_mmio_serial(&info, image->arch.pgd);
303 if (result)
304 return result;
305
306 /*
307 * This must be last because the intermediate page table pages it
308 * allocates will not be control pages and may overlap the image.
309 */
310 return init_transition_pgtable(image, image->arch.pgd, control_page);
311 }
312
load_segments(void)313 static void load_segments(void)
314 {
315 __asm__ __volatile__ (
316 "\tmovl %0,%%ds\n"
317 "\tmovl %0,%%es\n"
318 "\tmovl %0,%%ss\n"
319 "\tmovl %0,%%fs\n"
320 "\tmovl %0,%%gs\n"
321 : : "a" (__KERNEL_DS) : "memory"
322 );
323 }
324
prepare_debug_idt(unsigned long control_page,unsigned long vec_ofs)325 static void prepare_debug_idt(unsigned long control_page, unsigned long vec_ofs)
326 {
327 gate_desc idtentry = { 0 };
328 int i;
329
330 idtentry.bits.p = 1;
331 idtentry.bits.type = GATE_TRAP;
332 idtentry.segment = __KERNEL_CS;
333 idtentry.offset_low = (control_page & 0xFFFF) + vec_ofs;
334 idtentry.offset_middle = (control_page >> 16) & 0xFFFF;
335 idtentry.offset_high = control_page >> 32;
336
337 for (i = 0; i < 16; i++) {
338 kexec_debug_idt[i] = idtentry;
339 idtentry.offset_low += KEXEC_DEBUG_EXC_HANDLER_SIZE;
340 }
341 }
342
machine_kexec_prepare(struct kimage * image)343 int machine_kexec_prepare(struct kimage *image)
344 {
345 void *control_page = page_address(image->control_code_page);
346 unsigned long reloc_start = (unsigned long)__relocate_kernel_start;
347 unsigned long reloc_end = (unsigned long)__relocate_kernel_end;
348 int result;
349
350 /*
351 * Some early TDX-capable platforms have an erratum. A kernel
352 * partial write (a write transaction of less than cacheline
353 * lands at memory controller) to TDX private memory poisons that
354 * memory, and a subsequent read triggers a machine check.
355 *
356 * On those platforms the old kernel must reset TDX private
357 * memory before jumping to the new kernel otherwise the new
358 * kernel may see unexpected machine check. For simplicity
359 * just fail kexec/kdump on those platforms.
360 */
361 if (boot_cpu_has_bug(X86_BUG_TDX_PW_MCE)) {
362 pr_info_once("Not allowed on platform with tdx_pw_mce bug\n");
363 return -EOPNOTSUPP;
364 }
365
366 /* Setup the identity mapped 64bit page table */
367 result = init_pgtable(image, __pa(control_page));
368 if (result)
369 return result;
370 kexec_va_control_page = (unsigned long)control_page;
371 kexec_pa_table_page = (unsigned long)__pa(image->arch.pgd);
372
373 if (image->type == KEXEC_TYPE_DEFAULT)
374 kexec_pa_swap_page = page_to_pfn(image->swap_page) << PAGE_SHIFT;
375
376 prepare_debug_idt((unsigned long)__pa(control_page),
377 (unsigned long)kexec_debug_exc_vectors - reloc_start);
378
379 __memcpy(control_page, __relocate_kernel_start, reloc_end - reloc_start);
380
381 set_memory_rox((unsigned long)control_page, 1);
382
383 return 0;
384 }
385
machine_kexec_cleanup(struct kimage * image)386 void machine_kexec_cleanup(struct kimage *image)
387 {
388 void *control_page = page_address(image->control_code_page);
389
390 set_memory_nx((unsigned long)control_page, 1);
391 set_memory_rw((unsigned long)control_page, 1);
392
393 free_transition_pgtable(image);
394 }
395
396 /*
397 * Do not allocate memory (or fail in any way) in machine_kexec().
398 * We are past the point of no return, committed to rebooting now.
399 */
machine_kexec(struct kimage * image)400 void __nocfi machine_kexec(struct kimage *image)
401 {
402 unsigned long reloc_start = (unsigned long)__relocate_kernel_start;
403 relocate_kernel_fn *relocate_kernel_ptr;
404 unsigned int relocate_kernel_flags;
405 int save_ftrace_enabled;
406 void *control_page;
407
408 #ifdef CONFIG_KEXEC_JUMP
409 if (image->preserve_context)
410 save_processor_state();
411 #endif
412
413 save_ftrace_enabled = __ftrace_enabled_save();
414
415 /* Interrupts aren't acceptable while we reboot */
416 local_irq_disable();
417 hw_breakpoint_disable();
418 cet_disable();
419
420 if (image->preserve_context) {
421 #ifdef CONFIG_X86_IO_APIC
422 /*
423 * We need to put APICs in legacy mode so that we can
424 * get timer interrupts in second kernel. kexec/kdump
425 * paths already have calls to restore_boot_irq_mode()
426 * in one form or other. kexec jump path also need one.
427 */
428 clear_IO_APIC();
429 restore_boot_irq_mode();
430 #endif
431 }
432
433 control_page = page_address(image->control_code_page);
434
435 /*
436 * Allow for the possibility that relocate_kernel might not be at
437 * the very start of the page.
438 */
439 relocate_kernel_ptr = control_page + (unsigned long)relocate_kernel - reloc_start;
440
441 relocate_kernel_flags = 0;
442 if (image->preserve_context)
443 relocate_kernel_flags |= RELOC_KERNEL_PRESERVE_CONTEXT;
444
445 /*
446 * This must be done before load_segments() since it resets
447 * GS to 0 and percpu data needs the correct GS to work.
448 */
449 if (this_cpu_read(cache_state_incoherent))
450 relocate_kernel_flags |= RELOC_KERNEL_CACHE_INCOHERENT;
451
452 /*
453 * The segment registers are funny things, they have both a
454 * visible and an invisible part. Whenever the visible part is
455 * set to a specific selector, the invisible part is loaded
456 * with from a table in memory. At no other time is the
457 * descriptor table in memory accessed.
458 *
459 * Take advantage of this here by force loading the segments,
460 * before the GDT is zapped with an invalid value.
461 *
462 * load_segments() resets GS to 0. Don't make any function call
463 * after here since call depth tracking uses percpu variables to
464 * operate (relocate_kernel() is explicitly ignored by call depth
465 * tracking).
466 */
467 load_segments();
468
469 /* now call it */
470 image->start = relocate_kernel_ptr((unsigned long)image->head,
471 virt_to_phys(control_page),
472 image->start,
473 relocate_kernel_flags);
474
475 #ifdef CONFIG_KEXEC_JUMP
476 if (image->preserve_context)
477 restore_processor_state();
478 #endif
479
480 __ftrace_enabled_restore(save_ftrace_enabled);
481 }
482 /*
483 * Handover to the next kernel, no CFI concern.
484 */
485 ANNOTATE_NOCFI_SYM(machine_kexec);
486
487 /* arch-dependent functionality related to kexec file-based syscall */
488
489 #ifdef CONFIG_KEXEC_FILE
490 /*
491 * Apply purgatory relocations.
492 *
493 * @pi: Purgatory to be relocated.
494 * @section: Section relocations applying to.
495 * @relsec: Section containing RELAs.
496 * @symtabsec: Corresponding symtab.
497 *
498 * TODO: Some of the code belongs to generic code. Move that in kexec.c.
499 */
arch_kexec_apply_relocations_add(struct purgatory_info * pi,Elf_Shdr * section,const Elf_Shdr * relsec,const Elf_Shdr * symtabsec)500 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
501 Elf_Shdr *section, const Elf_Shdr *relsec,
502 const Elf_Shdr *symtabsec)
503 {
504 unsigned int i;
505 Elf64_Rela *rel;
506 Elf64_Sym *sym;
507 void *location;
508 unsigned long address, sec_base, value;
509 const char *strtab, *name, *shstrtab;
510 const Elf_Shdr *sechdrs;
511
512 /* String & section header string table */
513 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
514 strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset;
515 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
516
517 rel = (void *)pi->ehdr + relsec->sh_offset;
518
519 pr_debug("Applying relocate section %s to %u\n",
520 shstrtab + relsec->sh_name, relsec->sh_info);
521
522 for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) {
523
524 /*
525 * rel[i].r_offset contains byte offset from beginning
526 * of section to the storage unit affected.
527 *
528 * This is location to update. This is temporary buffer
529 * where section is currently loaded. This will finally be
530 * loaded to a different address later, pointed to by
531 * ->sh_addr. kexec takes care of moving it
532 * (kexec_load_segment()).
533 */
534 location = pi->purgatory_buf;
535 location += section->sh_offset;
536 location += rel[i].r_offset;
537
538 /* Final address of the location */
539 address = section->sh_addr + rel[i].r_offset;
540
541 /*
542 * rel[i].r_info contains information about symbol table index
543 * w.r.t which relocation must be made and type of relocation
544 * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
545 * these respectively.
546 */
547 sym = (void *)pi->ehdr + symtabsec->sh_offset;
548 sym += ELF64_R_SYM(rel[i].r_info);
549
550 if (sym->st_name)
551 name = strtab + sym->st_name;
552 else
553 name = shstrtab + sechdrs[sym->st_shndx].sh_name;
554
555 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
556 name, sym->st_info, sym->st_shndx, sym->st_value,
557 sym->st_size);
558
559 if (sym->st_shndx == SHN_UNDEF) {
560 pr_err("Undefined symbol: %s\n", name);
561 return -ENOEXEC;
562 }
563
564 if (sym->st_shndx == SHN_COMMON) {
565 pr_err("symbol '%s' in common section\n", name);
566 return -ENOEXEC;
567 }
568
569 if (sym->st_shndx == SHN_ABS)
570 sec_base = 0;
571 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
572 pr_err("Invalid section %d for symbol %s\n",
573 sym->st_shndx, name);
574 return -ENOEXEC;
575 } else
576 sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
577
578 value = sym->st_value;
579 value += sec_base;
580 value += rel[i].r_addend;
581
582 switch (ELF64_R_TYPE(rel[i].r_info)) {
583 case R_X86_64_NONE:
584 break;
585 case R_X86_64_64:
586 *(u64 *)location = value;
587 break;
588 case R_X86_64_32:
589 *(u32 *)location = value;
590 if (value != *(u32 *)location)
591 goto overflow;
592 break;
593 case R_X86_64_32S:
594 *(s32 *)location = value;
595 if ((s64)value != *(s32 *)location)
596 goto overflow;
597 break;
598 case R_X86_64_PC32:
599 case R_X86_64_PLT32:
600 value -= (u64)address;
601 *(u32 *)location = value;
602 break;
603 default:
604 pr_err("Unknown rela relocation: %llu\n",
605 ELF64_R_TYPE(rel[i].r_info));
606 return -ENOEXEC;
607 }
608 }
609 return 0;
610
611 overflow:
612 pr_err("Overflow in relocation type %d value 0x%lx\n",
613 (int)ELF64_R_TYPE(rel[i].r_info), value);
614 return -ENOEXEC;
615 }
616
arch_kimage_file_post_load_cleanup(struct kimage * image)617 int arch_kimage_file_post_load_cleanup(struct kimage *image)
618 {
619 vfree(image->elf_headers);
620 image->elf_headers = NULL;
621 image->elf_headers_sz = 0;
622
623 return kexec_image_post_load_cleanup_default(image);
624 }
625 #endif /* CONFIG_KEXEC_FILE */
626
627 #ifdef CONFIG_CRASH_DUMP
628
629 static int
kexec_mark_range(unsigned long start,unsigned long end,bool protect)630 kexec_mark_range(unsigned long start, unsigned long end, bool protect)
631 {
632 struct page *page;
633 unsigned int nr_pages;
634
635 /*
636 * For physical range: [start, end]. We must skip the unassigned
637 * crashk resource with zero-valued "end" member.
638 */
639 if (!end || start > end)
640 return 0;
641
642 page = pfn_to_page(start >> PAGE_SHIFT);
643 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
644 if (protect)
645 return set_pages_ro(page, nr_pages);
646 else
647 return set_pages_rw(page, nr_pages);
648 }
649
kexec_mark_crashkres(bool protect)650 static void kexec_mark_crashkres(bool protect)
651 {
652 unsigned long control;
653
654 kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
655
656 /* Don't touch the control code page used in crash_kexec().*/
657 control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
658 kexec_mark_range(crashk_res.start, control - 1, protect);
659 control += KEXEC_CONTROL_PAGE_SIZE;
660 kexec_mark_range(control, crashk_res.end, protect);
661 }
662
663 /* make the memory storing dm crypt keys in/accessible */
kexec_mark_dm_crypt_keys(bool protect)664 static void kexec_mark_dm_crypt_keys(bool protect)
665 {
666 unsigned long start_paddr, end_paddr;
667 unsigned int nr_pages;
668
669 if (kexec_crash_image->dm_crypt_keys_addr) {
670 start_paddr = kexec_crash_image->dm_crypt_keys_addr;
671 end_paddr = start_paddr + kexec_crash_image->dm_crypt_keys_sz - 1;
672 nr_pages = (PAGE_ALIGN(end_paddr) - PAGE_ALIGN_DOWN(start_paddr))/PAGE_SIZE;
673 if (protect)
674 set_memory_np((unsigned long)phys_to_virt(start_paddr), nr_pages);
675 else
676 __set_memory_prot(
677 (unsigned long)phys_to_virt(start_paddr),
678 nr_pages,
679 __pgprot(_PAGE_PRESENT | _PAGE_NX | _PAGE_RW));
680 }
681 }
682
arch_kexec_protect_crashkres(void)683 void arch_kexec_protect_crashkres(void)
684 {
685 kexec_mark_crashkres(true);
686 kexec_mark_dm_crypt_keys(true);
687 }
688
arch_kexec_unprotect_crashkres(void)689 void arch_kexec_unprotect_crashkres(void)
690 {
691 kexec_mark_dm_crypt_keys(false);
692 kexec_mark_crashkres(false);
693 }
694 #endif
695
696 /*
697 * During a traditional boot under SME, SME will encrypt the kernel,
698 * so the SME kexec kernel also needs to be un-encrypted in order to
699 * replicate a normal SME boot.
700 *
701 * During a traditional boot under SEV, the kernel has already been
702 * loaded encrypted, so the SEV kexec kernel needs to be encrypted in
703 * order to replicate a normal SEV boot.
704 */
arch_kexec_post_alloc_pages(void * vaddr,unsigned int pages,gfp_t gfp)705 int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)
706 {
707 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
708 return 0;
709
710 /*
711 * If host memory encryption is active we need to be sure that kexec
712 * pages are not encrypted because when we boot to the new kernel the
713 * pages won't be accessed encrypted (initially).
714 */
715 return set_memory_decrypted((unsigned long)vaddr, pages);
716 }
717
arch_kexec_pre_free_pages(void * vaddr,unsigned int pages)718 void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)
719 {
720 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
721 return;
722
723 /*
724 * If host memory encryption is active we need to reset the pages back
725 * to being an encrypted mapping before freeing them.
726 */
727 set_memory_encrypted((unsigned long)vaddr, pages);
728 }
729