140b0b3f8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
29a163ed8SThomas Gleixner /*
3835c34a1SDave Jones * handle transition of Linux booting another kernel
49a163ed8SThomas Gleixner * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
59a163ed8SThomas Gleixner */
69a163ed8SThomas Gleixner
79a163ed8SThomas Gleixner #include <linux/mm.h>
89a163ed8SThomas Gleixner #include <linux/kexec.h>
99a163ed8SThomas Gleixner #include <linux/delay.h>
10fd59d231SKen'ichi Ohmichi #include <linux/numa.h>
11f43fdad8SIngo Molnar #include <linux/ftrace.h>
123122c331SHuang Ying #include <linux/suspend.h>
1392be3d6bSHuang Ying #include <linux/gfp.h>
14fef3a7a1SHuang Ying #include <linux/io.h>
15f43fdad8SIngo Molnar
169a163ed8SThomas Gleixner #include <asm/pgalloc.h>
179a163ed8SThomas Gleixner #include <asm/tlbflush.h>
189a163ed8SThomas Gleixner #include <asm/mmu_context.h>
199a163ed8SThomas Gleixner #include <asm/apic.h>
208643e28dSJiang Liu #include <asm/io_apic.h>
219a163ed8SThomas Gleixner #include <asm/cpufeature.h>
229a163ed8SThomas Gleixner #include <asm/desc.h>
23d1163651SLaura Abbott #include <asm/set_memory.h>
2417f557e5SK.Prasad #include <asm/debugreg.h>
259a163ed8SThomas Gleixner
load_segments(void)269a163ed8SThomas Gleixner static void load_segments(void)
279a163ed8SThomas Gleixner {
289a163ed8SThomas Gleixner #define __STR(X) #X
299a163ed8SThomas Gleixner #define STR(X) __STR(X)
309a163ed8SThomas Gleixner
319a163ed8SThomas Gleixner __asm__ __volatile__ (
329a163ed8SThomas Gleixner "\tljmp $"STR(__KERNEL_CS)",$1f\n"
339a163ed8SThomas Gleixner "\t1:\n"
349a163ed8SThomas Gleixner "\tmovl $"STR(__KERNEL_DS)",%%eax\n"
359a163ed8SThomas Gleixner "\tmovl %%eax,%%ds\n"
369a163ed8SThomas Gleixner "\tmovl %%eax,%%es\n"
379a163ed8SThomas Gleixner "\tmovl %%eax,%%ss\n"
389a163ed8SThomas Gleixner : : : "eax", "memory");
399a163ed8SThomas Gleixner #undef STR
409a163ed8SThomas Gleixner #undef __STR
419a163ed8SThomas Gleixner }
429a163ed8SThomas Gleixner
machine_kexec_free_page_tables(struct kimage * image)4392be3d6bSHuang Ying static void machine_kexec_free_page_tables(struct kimage *image)
4492be3d6bSHuang Ying {
45ca38dc8fSJoerg Roedel free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER);
46a466ef76STetsuo Handa image->arch.pgd = NULL;
4792be3d6bSHuang Ying #ifdef CONFIG_X86_PAE
4892be3d6bSHuang Ying free_page((unsigned long)image->arch.pmd0);
49a466ef76STetsuo Handa image->arch.pmd0 = NULL;
5092be3d6bSHuang Ying free_page((unsigned long)image->arch.pmd1);
51a466ef76STetsuo Handa image->arch.pmd1 = NULL;
5292be3d6bSHuang Ying #endif
5392be3d6bSHuang Ying free_page((unsigned long)image->arch.pte0);
54a466ef76STetsuo Handa image->arch.pte0 = NULL;
5592be3d6bSHuang Ying free_page((unsigned long)image->arch.pte1);
56a466ef76STetsuo Handa image->arch.pte1 = NULL;
5792be3d6bSHuang Ying }
5892be3d6bSHuang Ying
machine_kexec_alloc_page_tables(struct kimage * image)5992be3d6bSHuang Ying static int machine_kexec_alloc_page_tables(struct kimage *image)
6092be3d6bSHuang Ying {
61ca38dc8fSJoerg Roedel image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
62ca38dc8fSJoerg Roedel PGD_ALLOCATION_ORDER);
6392be3d6bSHuang Ying #ifdef CONFIG_X86_PAE
6492be3d6bSHuang Ying image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
6592be3d6bSHuang Ying image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL);
6692be3d6bSHuang Ying #endif
6792be3d6bSHuang Ying image->arch.pte0 = (pte_t *)get_zeroed_page(GFP_KERNEL);
6892be3d6bSHuang Ying image->arch.pte1 = (pte_t *)get_zeroed_page(GFP_KERNEL);
6992be3d6bSHuang Ying if (!image->arch.pgd ||
7092be3d6bSHuang Ying #ifdef CONFIG_X86_PAE
7192be3d6bSHuang Ying !image->arch.pmd0 || !image->arch.pmd1 ||
7292be3d6bSHuang Ying #endif
7392be3d6bSHuang Ying !image->arch.pte0 || !image->arch.pte1) {
7492be3d6bSHuang Ying return -ENOMEM;
7592be3d6bSHuang Ying }
7692be3d6bSHuang Ying return 0;
7792be3d6bSHuang Ying }
7892be3d6bSHuang Ying
machine_kexec_page_table_set_one(pgd_t * pgd,pmd_t * pmd,pte_t * pte,unsigned long vaddr,unsigned long paddr)799868ee63SHuang Ying static void machine_kexec_page_table_set_one(
809868ee63SHuang Ying pgd_t *pgd, pmd_t *pmd, pte_t *pte,
819868ee63SHuang Ying unsigned long vaddr, unsigned long paddr)
829868ee63SHuang Ying {
837f689041SKirill A. Shutemov p4d_t *p4d;
849868ee63SHuang Ying pud_t *pud;
859868ee63SHuang Ying
869868ee63SHuang Ying pgd += pgd_index(vaddr);
879868ee63SHuang Ying #ifdef CONFIG_X86_PAE
889868ee63SHuang Ying if (!(pgd_val(*pgd) & _PAGE_PRESENT))
899868ee63SHuang Ying set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT));
909868ee63SHuang Ying #endif
917f689041SKirill A. Shutemov p4d = p4d_offset(pgd, vaddr);
927f689041SKirill A. Shutemov pud = pud_offset(p4d, vaddr);
939868ee63SHuang Ying pmd = pmd_offset(pud, vaddr);
949868ee63SHuang Ying if (!(pmd_val(*pmd) & _PAGE_PRESENT))
959868ee63SHuang Ying set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
969868ee63SHuang Ying pte = pte_offset_kernel(pmd, vaddr);
979868ee63SHuang Ying set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
989868ee63SHuang Ying }
999868ee63SHuang Ying
machine_kexec_prepare_page_tables(struct kimage * image)1009868ee63SHuang Ying static void machine_kexec_prepare_page_tables(struct kimage *image)
1019868ee63SHuang Ying {
1029868ee63SHuang Ying void *control_page;
103fc6fcdfbSHannes Eder pmd_t *pmd = NULL;
1049868ee63SHuang Ying
1059868ee63SHuang Ying control_page = page_address(image->control_code_page);
1069868ee63SHuang Ying #ifdef CONFIG_X86_PAE
1079868ee63SHuang Ying pmd = image->arch.pmd0;
1089868ee63SHuang Ying #endif
1099868ee63SHuang Ying machine_kexec_page_table_set_one(
1109868ee63SHuang Ying image->arch.pgd, pmd, image->arch.pte0,
1119868ee63SHuang Ying (unsigned long)control_page, __pa(control_page));
1129868ee63SHuang Ying #ifdef CONFIG_X86_PAE
1139868ee63SHuang Ying pmd = image->arch.pmd1;
1149868ee63SHuang Ying #endif
1159868ee63SHuang Ying machine_kexec_page_table_set_one(
1169868ee63SHuang Ying image->arch.pgd, pmd, image->arch.pte1,
1179868ee63SHuang Ying __pa(control_page), __pa(control_page));
1189868ee63SHuang Ying }
1199868ee63SHuang Ying
1209a163ed8SThomas Gleixner /*
1219a163ed8SThomas Gleixner * A architecture hook called to validate the
1229a163ed8SThomas Gleixner * proposed image and prepare the control pages
123163f6876SHuang Ying * as needed. The pages for KEXEC_CONTROL_PAGE_SIZE
1249a163ed8SThomas Gleixner * have been allocated, but the segments have yet
1259a163ed8SThomas Gleixner * been copied into the kernel.
1269a163ed8SThomas Gleixner *
1279a163ed8SThomas Gleixner * Do what every setup is needed on image and the
1289a163ed8SThomas Gleixner * reboot code buffer to allow us to avoid allocations
1299a163ed8SThomas Gleixner * later.
1309a163ed8SThomas Gleixner *
13192be3d6bSHuang Ying * - Make control page executable.
13292be3d6bSHuang Ying * - Allocate page tables
1339868ee63SHuang Ying * - Setup page tables
1349a163ed8SThomas Gleixner */
machine_kexec_prepare(struct kimage * image)1359a163ed8SThomas Gleixner int machine_kexec_prepare(struct kimage *image)
1369a163ed8SThomas Gleixner {
1379868ee63SHuang Ying int error;
1389868ee63SHuang Ying
139185be151SChristoph Hellwig set_memory_x((unsigned long)page_address(image->control_code_page), 1);
1409868ee63SHuang Ying error = machine_kexec_alloc_page_tables(image);
1419868ee63SHuang Ying if (error)
1429868ee63SHuang Ying return error;
1439868ee63SHuang Ying machine_kexec_prepare_page_tables(image);
1449868ee63SHuang Ying return 0;
1459a163ed8SThomas Gleixner }
1469a163ed8SThomas Gleixner
1479a163ed8SThomas Gleixner /*
1489a163ed8SThomas Gleixner * Undo anything leftover by machine_kexec_prepare
1499a163ed8SThomas Gleixner * when an image is freed.
1509a163ed8SThomas Gleixner */
machine_kexec_cleanup(struct kimage * image)1519a163ed8SThomas Gleixner void machine_kexec_cleanup(struct kimage *image)
1529a163ed8SThomas Gleixner {
153185be151SChristoph Hellwig set_memory_nx((unsigned long)page_address(image->control_code_page), 1);
15492be3d6bSHuang Ying machine_kexec_free_page_tables(image);
1559a163ed8SThomas Gleixner }
1569a163ed8SThomas Gleixner
1579a163ed8SThomas Gleixner /*
1589a163ed8SThomas Gleixner * Do not allocate memory (or fail in any way) in machine_kexec().
1599a163ed8SThomas Gleixner * We are past the point of no return, committed to rebooting now.
1609a163ed8SThomas Gleixner */
machine_kexec(struct kimage * image)1613ab83521SHuang Ying void machine_kexec(struct kimage *image)
1629a163ed8SThomas Gleixner {
1639a163ed8SThomas Gleixner unsigned long page_list[PAGES_NR];
1649a163ed8SThomas Gleixner void *control_page;
1653122c331SHuang Ying int save_ftrace_enabled;
1663ab83521SHuang Ying asmlinkage unsigned long
1673ab83521SHuang Ying (*relocate_kernel_ptr)(unsigned long indirection_page,
1683ab83521SHuang Ying unsigned long control_page,
1693ab83521SHuang Ying unsigned long start_address,
1703ab83521SHuang Ying unsigned int has_pae,
1713ab83521SHuang Ying unsigned int preserve_context);
1729a163ed8SThomas Gleixner
1733122c331SHuang Ying #ifdef CONFIG_KEXEC_JUMP
1746407df5cSHuang Ying if (image->preserve_context)
1753122c331SHuang Ying save_processor_state();
1763122c331SHuang Ying #endif
1773122c331SHuang Ying
1783122c331SHuang Ying save_ftrace_enabled = __ftrace_enabled_save();
179f43fdad8SIngo Molnar
1809a163ed8SThomas Gleixner /* Interrupts aren't acceptable while we reboot */
1819a163ed8SThomas Gleixner local_irq_disable();
18217f557e5SK.Prasad hw_breakpoint_disable();
1839a163ed8SThomas Gleixner
18489081d17SHuang Ying if (image->preserve_context) {
18589081d17SHuang Ying #ifdef CONFIG_X86_IO_APIC
186fef3a7a1SHuang Ying /*
187fef3a7a1SHuang Ying * We need to put APICs in legacy mode so that we can
18889081d17SHuang Ying * get timer interrupts in second kernel. kexec/kdump
18950374b96SBaoquan He * paths already have calls to restore_boot_irq_mode()
19050374b96SBaoquan He * in one form or other. kexec jump path also need one.
19189081d17SHuang Ying */
1923c9e76dbSBaoquan He clear_IO_APIC();
1933c9e76dbSBaoquan He restore_boot_irq_mode();
19489081d17SHuang Ying #endif
19589081d17SHuang Ying }
19689081d17SHuang Ying
1979a163ed8SThomas Gleixner control_page = page_address(image->control_code_page);
198fb45daa6SHuang Ying memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
1999a163ed8SThomas Gleixner
2003ab83521SHuang Ying relocate_kernel_ptr = control_page;
2019a163ed8SThomas Gleixner page_list[PA_CONTROL_PAGE] = __pa(control_page);
2023ab83521SHuang Ying page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
20392be3d6bSHuang Ying page_list[PA_PGD] = __pa(image->arch.pgd);
204e7706fc6SKen'ichi Ohmichi
205e7706fc6SKen'ichi Ohmichi if (image->type == KEXEC_TYPE_DEFAULT)
206e7706fc6SKen'ichi Ohmichi page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
207e7706fc6SKen'ichi Ohmichi << PAGE_SHIFT);
2089a163ed8SThomas Gleixner
209fef3a7a1SHuang Ying /*
210fef3a7a1SHuang Ying * The segment registers are funny things, they have both a
2119a163ed8SThomas Gleixner * visible and an invisible part. Whenever the visible part is
2129a163ed8SThomas Gleixner * set to a specific selector, the invisible part is loaded
2139a163ed8SThomas Gleixner * with from a table in memory. At no other time is the
2149a163ed8SThomas Gleixner * descriptor table in memory accessed.
2159a163ed8SThomas Gleixner *
2169a163ed8SThomas Gleixner * I take advantage of this here by force loading the
2179a163ed8SThomas Gleixner * segments, before I zap the gdt with an invalid value.
2189a163ed8SThomas Gleixner */
2199a163ed8SThomas Gleixner load_segments();
220fef3a7a1SHuang Ying /*
221fef3a7a1SHuang Ying * The gdt & idt are now invalid.
2229a163ed8SThomas Gleixner * If you want to load them you must set up your own idt & gdt.
2239a163ed8SThomas Gleixner */
224*056c52f5SH. Peter Anvin (Intel) native_idt_invalidate();
225*056c52f5SH. Peter Anvin (Intel) native_gdt_invalidate();
2269a163ed8SThomas Gleixner
2279a163ed8SThomas Gleixner /* now call it */
2283ab83521SHuang Ying image->start = relocate_kernel_ptr((unsigned long)image->head,
2293ab83521SHuang Ying (unsigned long)page_list,
230c8128cceSDave Hansen image->start,
231c8128cceSDave Hansen boot_cpu_has(X86_FEATURE_PAE),
2323ab83521SHuang Ying image->preserve_context);
2333122c331SHuang Ying
2343122c331SHuang Ying #ifdef CONFIG_KEXEC_JUMP
2356407df5cSHuang Ying if (image->preserve_context)
2363122c331SHuang Ying restore_processor_state();
2373122c331SHuang Ying #endif
2383122c331SHuang Ying
2393122c331SHuang Ying __ftrace_enabled_restore(save_ftrace_enabled);
2409a163ed8SThomas Gleixner }
241