1*5a67da1fSArd Biesheuvel/* SPDX-License-Identifier: GPL-2.0 */ 2*5a67da1fSArd Biesheuvel 3*5a67da1fSArd Biesheuvel#include <linux/linkage.h> 4*5a67da1fSArd Biesheuvel#include <asm/segment.h> 5*5a67da1fSArd Biesheuvel#include <asm/boot.h> 6*5a67da1fSArd Biesheuvel#include <asm/msr.h> 7*5a67da1fSArd Biesheuvel#include <asm/processor-flags.h> 8*5a67da1fSArd Biesheuvel 9*5a67da1fSArd Biesheuvel/* 10*5a67da1fSArd Biesheuvel * This is the 32-bit trampoline that will be copied over to low memory. It 11*5a67da1fSArd Biesheuvel * will be called using the ordinary 64-bit calling convention from code 12*5a67da1fSArd Biesheuvel * running in 64-bit mode. 13*5a67da1fSArd Biesheuvel * 14*5a67da1fSArd Biesheuvel * Return address is at the top of the stack (might be above 4G). 15*5a67da1fSArd Biesheuvel * The first argument (EDI) contains the address of the temporary PGD level 16*5a67da1fSArd Biesheuvel * page table in 32-bit addressable memory which will be programmed into 17*5a67da1fSArd Biesheuvel * register CR3. 18*5a67da1fSArd Biesheuvel */ 19*5a67da1fSArd Biesheuvel 20*5a67da1fSArd Biesheuvel .section ".rodata", "a", @progbits 21*5a67da1fSArd BiesheuvelSYM_CODE_START(trampoline_32bit_src) 22*5a67da1fSArd Biesheuvel /* 23*5a67da1fSArd Biesheuvel * Preserve callee save 64-bit registers on the stack: this is 24*5a67da1fSArd Biesheuvel * necessary because the architecture does not guarantee that GPRs will 25*5a67da1fSArd Biesheuvel * retain their full 64-bit values across a 32-bit mode switch. 26*5a67da1fSArd Biesheuvel */ 27*5a67da1fSArd Biesheuvel pushq %r15 28*5a67da1fSArd Biesheuvel pushq %r14 29*5a67da1fSArd Biesheuvel pushq %r13 30*5a67da1fSArd Biesheuvel pushq %r12 31*5a67da1fSArd Biesheuvel pushq %rbp 32*5a67da1fSArd Biesheuvel pushq %rbx 33*5a67da1fSArd Biesheuvel 34*5a67da1fSArd Biesheuvel /* Preserve top half of RSP in a legacy mode GPR to avoid truncation */ 35*5a67da1fSArd Biesheuvel movq %rsp, %rbx 36*5a67da1fSArd Biesheuvel shrq $32, %rbx 37*5a67da1fSArd Biesheuvel 38*5a67da1fSArd Biesheuvel /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */ 39*5a67da1fSArd Biesheuvel pushq $__KERNEL32_CS 40*5a67da1fSArd Biesheuvel leaq 0f(%rip), %rax 41*5a67da1fSArd Biesheuvel pushq %rax 42*5a67da1fSArd Biesheuvel lretq 43*5a67da1fSArd Biesheuvel 44*5a67da1fSArd Biesheuvel /* 45*5a67da1fSArd Biesheuvel * The 32-bit code below will do a far jump back to long mode and end 46*5a67da1fSArd Biesheuvel * up here after reconfiguring the number of paging levels. First, the 47*5a67da1fSArd Biesheuvel * stack pointer needs to be restored to its full 64-bit value before 48*5a67da1fSArd Biesheuvel * the callee save register contents can be popped from the stack. 49*5a67da1fSArd Biesheuvel */ 50*5a67da1fSArd Biesheuvel.Lret: 51*5a67da1fSArd Biesheuvel shlq $32, %rbx 52*5a67da1fSArd Biesheuvel orq %rbx, %rsp 53*5a67da1fSArd Biesheuvel 54*5a67da1fSArd Biesheuvel /* Restore the preserved 64-bit registers */ 55*5a67da1fSArd Biesheuvel popq %rbx 56*5a67da1fSArd Biesheuvel popq %rbp 57*5a67da1fSArd Biesheuvel popq %r12 58*5a67da1fSArd Biesheuvel popq %r13 59*5a67da1fSArd Biesheuvel popq %r14 60*5a67da1fSArd Biesheuvel popq %r15 61*5a67da1fSArd Biesheuvel retq 62*5a67da1fSArd Biesheuvel 63*5a67da1fSArd Biesheuvel .code32 64*5a67da1fSArd Biesheuvel0: 65*5a67da1fSArd Biesheuvel /* Disable paging */ 66*5a67da1fSArd Biesheuvel movl %cr0, %eax 67*5a67da1fSArd Biesheuvel btrl $X86_CR0_PG_BIT, %eax 68*5a67da1fSArd Biesheuvel movl %eax, %cr0 69*5a67da1fSArd Biesheuvel 70*5a67da1fSArd Biesheuvel /* Point CR3 to the trampoline's new top level page table */ 71*5a67da1fSArd Biesheuvel movl %edi, %cr3 72*5a67da1fSArd Biesheuvel 73*5a67da1fSArd Biesheuvel /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */ 74*5a67da1fSArd Biesheuvel movl $MSR_EFER, %ecx 75*5a67da1fSArd Biesheuvel rdmsr 76*5a67da1fSArd Biesheuvel btsl $_EFER_LME, %eax 77*5a67da1fSArd Biesheuvel /* Avoid writing EFER if no change was made (for TDX guest) */ 78*5a67da1fSArd Biesheuvel jc 1f 79*5a67da1fSArd Biesheuvel wrmsr 80*5a67da1fSArd Biesheuvel1: 81*5a67da1fSArd Biesheuvel /* Toggle CR4.LA57 */ 82*5a67da1fSArd Biesheuvel movl %cr4, %eax 83*5a67da1fSArd Biesheuvel btcl $X86_CR4_LA57_BIT, %eax 84*5a67da1fSArd Biesheuvel movl %eax, %cr4 85*5a67da1fSArd Biesheuvel 86*5a67da1fSArd Biesheuvel /* Enable paging again. */ 87*5a67da1fSArd Biesheuvel movl %cr0, %eax 88*5a67da1fSArd Biesheuvel btsl $X86_CR0_PG_BIT, %eax 89*5a67da1fSArd Biesheuvel movl %eax, %cr0 90*5a67da1fSArd Biesheuvel 91*5a67da1fSArd Biesheuvel /* 92*5a67da1fSArd Biesheuvel * Return to the 64-bit calling code using LJMP rather than LRET, to 93*5a67da1fSArd Biesheuvel * avoid the need for a 32-bit addressable stack. The destination 94*5a67da1fSArd Biesheuvel * address will be adjusted after the template code is copied into a 95*5a67da1fSArd Biesheuvel * 32-bit addressable buffer. 96*5a67da1fSArd Biesheuvel */ 97*5a67da1fSArd Biesheuvel.Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampoline_32bit_src) 98*5a67da1fSArd BiesheuvelSYM_CODE_END(trampoline_32bit_src) 99*5a67da1fSArd Biesheuvel 100*5a67da1fSArd Biesheuvel/* 101*5a67da1fSArd Biesheuvel * This symbol is placed right after trampoline_32bit_src() so its address can 102*5a67da1fSArd Biesheuvel * be used to infer the size of the trampoline code. 103*5a67da1fSArd Biesheuvel */ 104*5a67da1fSArd BiesheuvelSYM_DATA(trampoline_ljmp_imm_offset, .word .Ljmp + 1 - trampoline_32bit_src) 105*5a67da1fSArd Biesheuvel 106*5a67da1fSArd Biesheuvel /* 107*5a67da1fSArd Biesheuvel * The trampoline code has a size limit. 108*5a67da1fSArd Biesheuvel * Make sure we fail to compile if the trampoline code grows 109*5a67da1fSArd Biesheuvel * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes. 110*5a67da1fSArd Biesheuvel */ 111*5a67da1fSArd Biesheuvel .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE 112