1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * relocate_kernel.S - put the kernel image in place to boot 4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7#include <linux/linkage.h> 8#include <asm/page_types.h> 9#include <asm/kexec.h> 10#include <asm/processor-flags.h> 11#include <asm/pgtable_types.h> 12#include <asm/nospec-branch.h> 13#include <asm/unwind_hints.h> 14 15/* 16 * Must be relocatable PIC code callable as a C function 17 */ 18 19#define PTR(x) (x << 3) 20#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) 21 22/* 23 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE 24 * ~ control_page + PAGE_SIZE are used as data storage and stack for 25 * jumping back 26 */ 27#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) 28 29/* Minimal CPU state */ 30#define RSP DATA(0x0) 31#define CR0 DATA(0x8) 32#define CR3 DATA(0x10) 33#define CR4 DATA(0x18) 34 35/* other data */ 36#define CP_PA_TABLE_PAGE DATA(0x20) 37#define CP_PA_SWAP_PAGE DATA(0x28) 38#define CP_PA_BACKUP_PAGES_MAP DATA(0x30) 39 40 .text 41 .align PAGE_SIZE 42 .code64 43SYM_CODE_START_NOALIGN(relocate_kernel) 44 UNWIND_HINT_EMPTY 45 /* 46 * %rdi indirection_page 47 * %rsi page_list 48 * %rdx start address 49 * %rcx preserve_context 50 * %r8 sme_active 51 */ 52 53 /* Save the CPU context, used for jumping back */ 54 pushq %rbx 55 pushq %rbp 56 pushq %r12 57 pushq %r13 58 pushq %r14 59 pushq %r15 60 pushf 61 62 movq PTR(VA_CONTROL_PAGE)(%rsi), %r11 63 movq %rsp, RSP(%r11) 64 movq %cr0, %rax 65 movq %rax, CR0(%r11) 66 movq %cr3, %rax 67 movq %rax, CR3(%r11) 68 movq %cr4, %rax 69 movq %rax, CR4(%r11) 70 71 /* Save CR4. Required to enable the right paging mode later. */ 72 movq %rax, %r13 73 74 /* zero out flags, and disable interrupts */ 75 pushq $0 76 popfq 77 78 /* Save SME active flag */ 79 movq %r8, %r12 80 81 /* 82 * get physical address of control page now 83 * this is impossible after page table switch 84 */ 85 movq PTR(PA_CONTROL_PAGE)(%rsi), %r8 86 87 /* get physical address of page table now too */ 88 movq PTR(PA_TABLE_PAGE)(%rsi), %r9 89 90 /* get physical address of swap page now */ 91 movq PTR(PA_SWAP_PAGE)(%rsi), %r10 92 93 /* save some information for jumping back */ 94 movq %r9, CP_PA_TABLE_PAGE(%r11) 95 movq %r10, CP_PA_SWAP_PAGE(%r11) 96 movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11) 97 98 /* Switch to the identity mapped page tables */ 99 movq %r9, %cr3 100 101 /* setup a new stack at the end of the physical control page */ 102 lea PAGE_SIZE(%r8), %rsp 103 104 /* jump to identity mapped page */ 105 addq $(identity_mapped - relocate_kernel), %r8 106 pushq %r8 107 ret 108SYM_CODE_END(relocate_kernel) 109 110SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) 111 UNWIND_HINT_EMPTY 112 /* set return address to 0 if not preserving context */ 113 pushq $0 114 /* store the start address on the stack */ 115 pushq %rdx 116 117 /* 118 * Set cr0 to a known state: 119 * - Paging enabled 120 * - Alignment check disabled 121 * - Write protect disabled 122 * - No task switch 123 * - Don't do FP software emulation. 124 * - Proctected mode enabled 125 */ 126 movq %cr0, %rax 127 andq $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax 128 orl $(X86_CR0_PG | X86_CR0_PE), %eax 129 movq %rax, %cr0 130 131 /* 132 * Set cr4 to a known state: 133 * - physical address extension enabled 134 * - 5-level paging, if it was enabled before 135 */ 136 movl $X86_CR4_PAE, %eax 137 testq $X86_CR4_LA57, %r13 138 jz 1f 139 orl $X86_CR4_LA57, %eax 1401: 141 movq %rax, %cr4 142 143 jmp 1f 1441: 145 146 /* Flush the TLB (needed?) */ 147 movq %r9, %cr3 148 149 /* 150 * If SME is active, there could be old encrypted cache line 151 * entries that will conflict with the now unencrypted memory 152 * used by kexec. Flush the caches before copying the kernel. 153 */ 154 testq %r12, %r12 155 jz 1f 156 wbinvd 1571: 158 159 movq %rcx, %r11 160 call swap_pages 161 162 /* 163 * To be certain of avoiding problems with self-modifying code 164 * I need to execute a serializing instruction here. 165 * So I flush the TLB by reloading %cr3 here, it's handy, 166 * and not processor dependent. 167 */ 168 movq %cr3, %rax 169 movq %rax, %cr3 170 171 /* 172 * set all of the registers to known values 173 * leave %rsp alone 174 */ 175 176 testq %r11, %r11 177 jnz 1f 178 xorl %eax, %eax 179 xorl %ebx, %ebx 180 xorl %ecx, %ecx 181 xorl %edx, %edx 182 xorl %esi, %esi 183 xorl %edi, %edi 184 xorl %ebp, %ebp 185 xorl %r8d, %r8d 186 xorl %r9d, %r9d 187 xorl %r10d, %r10d 188 xorl %r11d, %r11d 189 xorl %r12d, %r12d 190 xorl %r13d, %r13d 191 xorl %r14d, %r14d 192 xorl %r15d, %r15d 193 194 ret 195 1961: 197 popq %rdx 198 leaq PAGE_SIZE(%r10), %rsp 199 ANNOTATE_RETPOLINE_SAFE 200 call *%rdx 201 202 /* get the re-entry point of the peer system */ 203 movq 0(%rsp), %rbp 204 leaq relocate_kernel(%rip), %r8 205 movq CP_PA_SWAP_PAGE(%r8), %r10 206 movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi 207 movq CP_PA_TABLE_PAGE(%r8), %rax 208 movq %rax, %cr3 209 lea PAGE_SIZE(%r8), %rsp 210 call swap_pages 211 movq $virtual_mapped, %rax 212 pushq %rax 213 ret 214SYM_CODE_END(identity_mapped) 215 216SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) 217 UNWIND_HINT_EMPTY 218 movq RSP(%r8), %rsp 219 movq CR4(%r8), %rax 220 movq %rax, %cr4 221 movq CR3(%r8), %rax 222 movq CR0(%r8), %r8 223 movq %rax, %cr3 224 movq %r8, %cr0 225 movq %rbp, %rax 226 227 popf 228 popq %r15 229 popq %r14 230 popq %r13 231 popq %r12 232 popq %rbp 233 popq %rbx 234 ret 235SYM_CODE_END(virtual_mapped) 236 237 /* Do the copies */ 238SYM_CODE_START_LOCAL_NOALIGN(swap_pages) 239 UNWIND_HINT_EMPTY 240 movq %rdi, %rcx /* Put the page_list in %rcx */ 241 xorl %edi, %edi 242 xorl %esi, %esi 243 jmp 1f 244 2450: /* top, read another word for the indirection page */ 246 247 movq (%rbx), %rcx 248 addq $8, %rbx 2491: 250 testb $0x1, %cl /* is it a destination page? */ 251 jz 2f 252 movq %rcx, %rdi 253 andq $0xfffffffffffff000, %rdi 254 jmp 0b 2552: 256 testb $0x2, %cl /* is it an indirection page? */ 257 jz 2f 258 movq %rcx, %rbx 259 andq $0xfffffffffffff000, %rbx 260 jmp 0b 2612: 262 testb $0x4, %cl /* is it the done indicator? */ 263 jz 2f 264 jmp 3f 2652: 266 testb $0x8, %cl /* is it the source indicator? */ 267 jz 0b /* Ignore it otherwise */ 268 movq %rcx, %rsi /* For ever source page do a copy */ 269 andq $0xfffffffffffff000, %rsi 270 271 movq %rdi, %rdx 272 movq %rsi, %rax 273 274 movq %r10, %rdi 275 movl $512, %ecx 276 rep ; movsq 277 278 movq %rax, %rdi 279 movq %rdx, %rsi 280 movl $512, %ecx 281 rep ; movsq 282 283 movq %rdx, %rdi 284 movq %r10, %rsi 285 movl $512, %ecx 286 rep ; movsq 287 288 lea PAGE_SIZE(%rax), %rsi 289 jmp 0b 2903: 291 ret 292SYM_CODE_END(swap_pages) 293 294 .globl kexec_control_code_size 295.set kexec_control_code_size, . - relocate_kernel 296