1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * relocate_kernel.S - put the kernel image in place to boot 4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7#include <linux/linkage.h> 8#include <asm/page_types.h> 9#include <asm/kexec.h> 10#include <asm/processor-flags.h> 11#include <asm/pgtable_types.h> 12#include <asm/nospec-branch.h> 13#include <asm/unwind_hints.h> 14 15/* 16 * Must be relocatable PIC code callable as a C function, in particular 17 * there must be a plain RET and not jump to return thunk. 18 */ 19 20#define PTR(x) (x << 3) 21#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) 22 23/* 24 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE 25 * ~ control_page + PAGE_SIZE are used as data storage and stack for 26 * jumping back 27 */ 28#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) 29 30/* Minimal CPU state */ 31#define RSP DATA(0x0) 32#define CR0 DATA(0x8) 33#define CR3 DATA(0x10) 34#define CR4 DATA(0x18) 35 36/* other data */ 37#define CP_PA_TABLE_PAGE DATA(0x20) 38#define CP_PA_SWAP_PAGE DATA(0x28) 39#define CP_PA_BACKUP_PAGES_MAP DATA(0x30) 40 41 .text 42 .align PAGE_SIZE 43 .code64 44SYM_CODE_START_NOALIGN(relocate_range) 45SYM_CODE_START_NOALIGN(relocate_kernel) 46 UNWIND_HINT_END_OF_STACK 47 ANNOTATE_NOENDBR 48 /* 49 * %rdi indirection_page 50 * %rsi page_list 51 * %rdx start address 52 * %rcx preserve_context 53 * %r8 host_mem_enc_active 54 */ 55 56 /* Save the CPU context, used for jumping back */ 57 pushq %rbx 58 pushq %rbp 59 pushq %r12 60 pushq %r13 61 pushq %r14 62 pushq %r15 63 pushf 64 65 movq PTR(VA_CONTROL_PAGE)(%rsi), %r11 66 movq %rsp, RSP(%r11) 67 movq %cr0, %rax 68 movq %rax, CR0(%r11) 69 movq %cr3, %rax 70 movq %rax, CR3(%r11) 71 movq %cr4, %rax 72 movq %rax, CR4(%r11) 73 74 /* Save CR4. Required to enable the right paging mode later. */ 75 movq %rax, %r13 76 77 /* zero out flags, and disable interrupts */ 78 pushq $0 79 popfq 80 81 /* Save SME active flag */ 82 movq %r8, %r12 83 84 /* 85 * get physical address of control page now 86 * this is impossible after page table switch 87 */ 88 movq PTR(PA_CONTROL_PAGE)(%rsi), %r8 89 90 /* get physical address of page table now too */ 91 movq PTR(PA_TABLE_PAGE)(%rsi), %r9 92 93 /* get physical address of swap page now */ 94 movq PTR(PA_SWAP_PAGE)(%rsi), %r10 95 96 /* save some information for jumping back */ 97 movq %r9, CP_PA_TABLE_PAGE(%r11) 98 movq %r10, CP_PA_SWAP_PAGE(%r11) 99 movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11) 100 101 /* Switch to the identity mapped page tables */ 102 movq %r9, %cr3 103 104 /* setup a new stack at the end of the physical control page */ 105 lea PAGE_SIZE(%r8), %rsp 106 107 /* jump to identity mapped page */ 108 addq $(identity_mapped - relocate_kernel), %r8 109 pushq %r8 110 ANNOTATE_UNRET_SAFE 111 ret 112 int3 113SYM_CODE_END(relocate_kernel) 114 115SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) 116 UNWIND_HINT_END_OF_STACK 117 /* set return address to 0 if not preserving context */ 118 pushq $0 119 /* store the start address on the stack */ 120 pushq %rdx 121 122 /* 123 * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP 124 * below. 125 */ 126 movq %cr4, %rax 127 andq $~(X86_CR4_CET), %rax 128 movq %rax, %cr4 129 130 /* 131 * Set cr0 to a known state: 132 * - Paging enabled 133 * - Alignment check disabled 134 * - Write protect disabled 135 * - No task switch 136 * - Don't do FP software emulation. 137 * - Protected mode enabled 138 */ 139 movq %cr0, %rax 140 andq $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax 141 orl $(X86_CR0_PG | X86_CR0_PE), %eax 142 movq %rax, %cr0 143 144 /* 145 * Set cr4 to a known state: 146 * - physical address extension enabled 147 * - 5-level paging, if it was enabled before 148 */ 149 movl $X86_CR4_PAE, %eax 150 testq $X86_CR4_LA57, %r13 151 jz 1f 152 orl $X86_CR4_LA57, %eax 1531: 154 movq %rax, %cr4 155 156 jmp 1f 1571: 158 159 /* Flush the TLB (needed?) */ 160 movq %r9, %cr3 161 162 /* 163 * If SME is active, there could be old encrypted cache line 164 * entries that will conflict with the now unencrypted memory 165 * used by kexec. Flush the caches before copying the kernel. 166 */ 167 testq %r12, %r12 168 jz 1f 169 wbinvd 1701: 171 172 movq %rcx, %r11 173 call swap_pages 174 175 /* 176 * To be certain of avoiding problems with self-modifying code 177 * I need to execute a serializing instruction here. 178 * So I flush the TLB by reloading %cr3 here, it's handy, 179 * and not processor dependent. 180 */ 181 movq %cr3, %rax 182 movq %rax, %cr3 183 184 /* 185 * set all of the registers to known values 186 * leave %rsp alone 187 */ 188 189 testq %r11, %r11 190 jnz 1f 191 xorl %eax, %eax 192 xorl %ebx, %ebx 193 xorl %ecx, %ecx 194 xorl %edx, %edx 195 xorl %esi, %esi 196 xorl %edi, %edi 197 xorl %ebp, %ebp 198 xorl %r8d, %r8d 199 xorl %r9d, %r9d 200 xorl %r10d, %r10d 201 xorl %r11d, %r11d 202 xorl %r12d, %r12d 203 xorl %r13d, %r13d 204 xorl %r14d, %r14d 205 xorl %r15d, %r15d 206 207 ANNOTATE_UNRET_SAFE 208 ret 209 int3 210 2111: 212 popq %rdx 213 leaq PAGE_SIZE(%r10), %rsp 214 ANNOTATE_RETPOLINE_SAFE 215 call *%rdx 216 217 /* get the re-entry point of the peer system */ 218 movq 0(%rsp), %rbp 219 leaq relocate_kernel(%rip), %r8 220 movq CP_PA_SWAP_PAGE(%r8), %r10 221 movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi 222 movq CP_PA_TABLE_PAGE(%r8), %rax 223 movq %rax, %cr3 224 lea PAGE_SIZE(%r8), %rsp 225 call swap_pages 226 movq $virtual_mapped, %rax 227 pushq %rax 228 ANNOTATE_UNRET_SAFE 229 ret 230 int3 231SYM_CODE_END(identity_mapped) 232 233SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) 234 UNWIND_HINT_END_OF_STACK 235 ANNOTATE_NOENDBR // RET target, above 236 movq RSP(%r8), %rsp 237 movq CR4(%r8), %rax 238 movq %rax, %cr4 239 movq CR3(%r8), %rax 240 movq CR0(%r8), %r8 241 movq %rax, %cr3 242 movq %r8, %cr0 243 movq %rbp, %rax 244 245 popf 246 popq %r15 247 popq %r14 248 popq %r13 249 popq %r12 250 popq %rbp 251 popq %rbx 252 ANNOTATE_UNRET_SAFE 253 ret 254 int3 255SYM_CODE_END(virtual_mapped) 256 257 /* Do the copies */ 258SYM_CODE_START_LOCAL_NOALIGN(swap_pages) 259 UNWIND_HINT_END_OF_STACK 260 movq %rdi, %rcx /* Put the page_list in %rcx */ 261 xorl %edi, %edi 262 xorl %esi, %esi 263 jmp 1f 264 2650: /* top, read another word for the indirection page */ 266 267 movq (%rbx), %rcx 268 addq $8, %rbx 2691: 270 testb $0x1, %cl /* is it a destination page? */ 271 jz 2f 272 movq %rcx, %rdi 273 andq $0xfffffffffffff000, %rdi 274 jmp 0b 2752: 276 testb $0x2, %cl /* is it an indirection page? */ 277 jz 2f 278 movq %rcx, %rbx 279 andq $0xfffffffffffff000, %rbx 280 jmp 0b 2812: 282 testb $0x4, %cl /* is it the done indicator? */ 283 jz 2f 284 jmp 3f 2852: 286 testb $0x8, %cl /* is it the source indicator? */ 287 jz 0b /* Ignore it otherwise */ 288 movq %rcx, %rsi /* For ever source page do a copy */ 289 andq $0xfffffffffffff000, %rsi 290 291 movq %rdi, %rdx 292 movq %rsi, %rax 293 294 movq %r10, %rdi 295 movl $512, %ecx 296 rep ; movsq 297 298 movq %rax, %rdi 299 movq %rdx, %rsi 300 movl $512, %ecx 301 rep ; movsq 302 303 movq %rdx, %rdi 304 movq %r10, %rsi 305 movl $512, %ecx 306 rep ; movsq 307 308 lea PAGE_SIZE(%rax), %rsi 309 jmp 0b 3103: 311 ANNOTATE_UNRET_SAFE 312 ret 313 int3 314SYM_CODE_END(swap_pages) 315 316 .skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc 317SYM_CODE_END(relocate_range); 318