1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * relocate_kernel.S - put the kernel image in place to boot 4 * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7#include <linux/linkage.h> 8#include <asm/page_types.h> 9#include <asm/kexec.h> 10#include <asm/nospec-branch.h> 11#include <asm/processor-flags.h> 12 13/* 14 * Must be relocatable PIC code callable as a C function, in particular 15 * there must be a plain RET and not jump to return thunk. 16 */ 17 18#define PTR(x) (x << 2) 19 20/* 21 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE 22 * ~ control_page + PAGE_SIZE are used as data storage and stack for 23 * jumping back 24 */ 25#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) 26 27/* Minimal CPU state */ 28#define ESP DATA(0x0) 29#define CR0 DATA(0x4) 30#define CR3 DATA(0x8) 31#define CR4 DATA(0xc) 32 33/* other data */ 34#define CP_VA_CONTROL_PAGE DATA(0x10) 35#define CP_PA_PGD DATA(0x14) 36#define CP_PA_SWAP_PAGE DATA(0x18) 37#define CP_PA_BACKUP_PAGES_MAP DATA(0x1c) 38 39 .text 40SYM_CODE_START_NOALIGN(relocate_kernel) 41 /* Save the CPU context, used for jumping back */ 42 43 pushl %ebx 44 pushl %esi 45 pushl %edi 46 pushl %ebp 47 pushf 48 49 movl 20+8(%esp), %ebp /* list of pages */ 50 movl PTR(VA_CONTROL_PAGE)(%ebp), %edi 51 movl %esp, ESP(%edi) 52 movl %cr0, %eax 53 movl %eax, CR0(%edi) 54 movl %cr3, %eax 55 movl %eax, CR3(%edi) 56 movl %cr4, %eax 57 movl %eax, CR4(%edi) 58 59 /* read the arguments and say goodbye to the stack */ 60 movl 20+4(%esp), %ebx /* page_list */ 61 movl 20+8(%esp), %ebp /* list of pages */ 62 movl 20+12(%esp), %edx /* start address */ 63 movl 20+16(%esp), %ecx /* cpu_has_pae */ 64 movl 20+20(%esp), %esi /* preserve_context */ 65 66 /* zero out flags, and disable interrupts */ 67 pushl $0 68 popfl 69 70 /* save some information for jumping back */ 71 movl PTR(VA_CONTROL_PAGE)(%ebp), %edi 72 movl %edi, CP_VA_CONTROL_PAGE(%edi) 73 movl PTR(PA_PGD)(%ebp), %eax 74 movl %eax, CP_PA_PGD(%edi) 75 movl PTR(PA_SWAP_PAGE)(%ebp), %eax 76 movl %eax, CP_PA_SWAP_PAGE(%edi) 77 movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi) 78 79 /* 80 * get physical address of control page now 81 * this is impossible after page table switch 82 */ 83 movl PTR(PA_CONTROL_PAGE)(%ebp), %edi 84 85 /* switch to new set of page tables */ 86 movl PTR(PA_PGD)(%ebp), %eax 87 movl %eax, %cr3 88 89 /* setup a new stack at the end of the physical control page */ 90 lea PAGE_SIZE(%edi), %esp 91 92 /* jump to identity mapped page */ 93 movl %edi, %eax 94 addl $(identity_mapped - relocate_kernel), %eax 95 pushl %eax 96 ANNOTATE_UNRET_SAFE 97 ret 98 int3 99SYM_CODE_END(relocate_kernel) 100 101SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) 102 /* set return address to 0 if not preserving context */ 103 pushl $0 104 /* store the start address on the stack */ 105 pushl %edx 106 107 /* 108 * Set cr0 to a known state: 109 * - Paging disabled 110 * - Alignment check disabled 111 * - Write protect disabled 112 * - No task switch 113 * - Don't do FP software emulation. 114 * - Protected mode enabled 115 */ 116 movl %cr0, %eax 117 andl $~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax 118 orl $(X86_CR0_PE), %eax 119 movl %eax, %cr0 120 121 /* clear cr4 if applicable */ 122 testl %ecx, %ecx 123 jz 1f 124 /* 125 * Set cr4 to a known state: 126 * Setting everything to zero seems safe. 127 */ 128 xorl %eax, %eax 129 movl %eax, %cr4 130 131 jmp 1f 1321: 133 134 /* Flush the TLB (needed?) */ 135 xorl %eax, %eax 136 movl %eax, %cr3 137 138 movl CP_PA_SWAP_PAGE(%edi), %eax 139 pushl %eax 140 pushl %ebx 141 call swap_pages 142 addl $8, %esp 143 144 /* 145 * To be certain of avoiding problems with self-modifying code 146 * I need to execute a serializing instruction here. 147 * So I flush the TLB, it's handy, and not processor dependent. 148 */ 149 xorl %eax, %eax 150 movl %eax, %cr3 151 152 /* 153 * set all of the registers to known values 154 * leave %esp alone 155 */ 156 157 testl %esi, %esi 158 jnz 1f 159 xorl %edi, %edi 160 xorl %eax, %eax 161 xorl %ebx, %ebx 162 xorl %ecx, %ecx 163 xorl %edx, %edx 164 xorl %esi, %esi 165 xorl %ebp, %ebp 166 ANNOTATE_UNRET_SAFE 167 ret 168 int3 1691: 170 popl %edx 171 movl CP_PA_SWAP_PAGE(%edi), %esp 172 addl $PAGE_SIZE, %esp 1732: 174 ANNOTATE_RETPOLINE_SAFE 175 call *%edx 176 177 /* get the re-entry point of the peer system */ 178 movl 0(%esp), %ebp 179 call 1f 1801: 181 popl %ebx 182 subl $(1b - relocate_kernel), %ebx 183 movl CP_VA_CONTROL_PAGE(%ebx), %edi 184 lea PAGE_SIZE(%ebx), %esp 185 movl CP_PA_SWAP_PAGE(%ebx), %eax 186 movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx 187 pushl %eax 188 pushl %edx 189 call swap_pages 190 addl $8, %esp 191 movl CP_PA_PGD(%ebx), %eax 192 movl %eax, %cr3 193 movl %cr0, %eax 194 orl $X86_CR0_PG, %eax 195 movl %eax, %cr0 196 lea PAGE_SIZE(%edi), %esp 197 movl %edi, %eax 198 addl $(virtual_mapped - relocate_kernel), %eax 199 pushl %eax 200 ANNOTATE_UNRET_SAFE 201 ret 202 int3 203SYM_CODE_END(identity_mapped) 204 205SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) 206 movl CR4(%edi), %eax 207 movl %eax, %cr4 208 movl CR3(%edi), %eax 209 movl %eax, %cr3 210 movl CR0(%edi), %eax 211 movl %eax, %cr0 212 movl ESP(%edi), %esp 213 movl %ebp, %eax 214 215 popf 216 popl %ebp 217 popl %edi 218 popl %esi 219 popl %ebx 220 ANNOTATE_UNRET_SAFE 221 ret 222 int3 223SYM_CODE_END(virtual_mapped) 224 225 /* Do the copies */ 226SYM_CODE_START_LOCAL_NOALIGN(swap_pages) 227 movl 8(%esp), %edx 228 movl 4(%esp), %ecx 229 pushl %ebp 230 pushl %ebx 231 pushl %edi 232 pushl %esi 233 movl %ecx, %ebx 234 jmp 1f 235 2360: /* top, read another word from the indirection page */ 237 movl (%ebx), %ecx 238 addl $4, %ebx 2391: 240 testb $0x1, %cl /* is it a destination page */ 241 jz 2f 242 movl %ecx, %edi 243 andl $0xfffff000, %edi 244 jmp 0b 2452: 246 testb $0x2, %cl /* is it an indirection page */ 247 jz 2f 248 movl %ecx, %ebx 249 andl $0xfffff000, %ebx 250 jmp 0b 2512: 252 testb $0x4, %cl /* is it the done indicator */ 253 jz 2f 254 jmp 3f 2552: 256 testb $0x8, %cl /* is it the source indicator */ 257 jz 0b /* Ignore it otherwise */ 258 movl %ecx, %esi /* For every source page do a copy */ 259 andl $0xfffff000, %esi 260 261 movl %edi, %eax 262 movl %esi, %ebp 263 264 movl %edx, %edi 265 movl $1024, %ecx 266 rep ; movsl 267 268 movl %ebp, %edi 269 movl %eax, %esi 270 movl $1024, %ecx 271 rep ; movsl 272 273 movl %eax, %edi 274 movl %edx, %esi 275 movl $1024, %ecx 276 rep ; movsl 277 278 lea PAGE_SIZE(%ebp), %esi 279 jmp 0b 2803: 281 popl %esi 282 popl %edi 283 popl %ebx 284 popl %ebp 285 ANNOTATE_UNRET_SAFE 286 ret 287 int3 288SYM_CODE_END(swap_pages) 289 290 .globl kexec_control_code_size 291.set kexec_control_code_size, . - relocate_kernel 292