1/*- 2 * Copyright (c) 2003 Peter Wemm <peter@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Copyright (c) 2020, 2021 The FreeBSD Foundation 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include <machine/asmacros.h> 34#include <machine/psl.h> 35#include <machine/pmap.h> 36#include <machine/specialreg.h> 37 38#include "assym.inc" 39 40/* 41 * Compiled KERNBASE location 42 */ 43 .globl kernbase, kernload 44 .set kernbase,KERNBASE 45 .set kernload,KERNLOAD 46 47#define BOOTSTACK_SIZE (PAGE_SIZE * KSTACK_PAGES) 48 49 .text 50/********************************************************************** 51 * 52 * This is where the loader trampoline start us, set the ball rolling... 53 * 54 * We are called with the stack looking like this: 55 * 0(%rsp) = 32 bit return address (cannot be used) 56 * 4(%rsp) = 32 bit modulep 57 * 8(%rsp) = 32 bit kernend 58 * 59 * We are already in long mode, on a 64 bit %cs and running at KERNBASE. 60 */ 61ENTRY(btext) 62 63 /* Don't trust what the loader gives for rflags. */ 64 pushq $PSL_KERNEL 65 popfq 66 67 /* Get onto a stack that we can trust - there is no going back now. */ 68 movq %rsp, %rbp 69 movq $bootstack,%rsp 70 71#ifdef KASAN 72 /* Bootstrap a shadow map for the boot stack. */ 73 movq $bootstack, %rdi 74 subq $BOOTSTACK_SIZE, %rdi 75 movq $BOOTSTACK_SIZE, %rsi 76 call kasan_init_early 77#endif 78 79 /* Grab metadata pointers from the loader. */ 80 movl 4(%rbp),%edi /* modulep (arg 1) */ 81 movl 8(%rbp),%esi /* kernend (arg 2) */ 82 xorq %rbp, %rbp 83 84 call hammer_time /* set up cpu for unix operation */ 85 movq %rax,%rsp /* set up kstack for mi_startup() */ 86 call mi_startup /* autoconfiguration, mountroot etc */ 870: hlt 88 jmp 0b 89 90/* 91 * void la57_trampoline(%rdi pml5) 92 * 93 * Entered in 4-level paging long mode on AP, hopefully returns alive in 94 * 5-level paging mode. The parameter is a pointer to a 5-level page 95 * table root. The passed 5-level page table, and the current 4-level page 96 * table, both must map the trampoline code page 1:1 physical, below 4G. 97 * The trampoline must be PIC because it is copied from kernel text into 98 * this page. 99 * 100 * The current paging level cannot be changed while paging is enabled, and 101 * paging cannot be disabled while in long mode. As consequence, code 102 * switches into the compat mode, then disables paging to descend into 103 * protected mode. There, the paging level bit CR4.LA57 can be changed, 104 * and code directly jumps back into long mode. 105 * 106 * Falling into the protected mode requires single-purpose GDT entries, 107 * which are provided by the private GDT. It is the caller's responsibility 108 * to 109 * - restore the GDT and %gsbase after the call 110 * - reset IDT back to long mode. 111 */ 112ENTRY(la57_trampoline) 113 movq %rsp,lst(%rip) /* save registers into memory */ 114 movq %rbx,lst+8(%rip) /* upper halves are not saved .. */ 115 movq %rbp,lst+0x10(%rip) /* by 64->32->64 switch */ 116 movq %cr4,%rax 117 orq $CR4_LA57,%rax /* 5-lvl %cr4 */ 118 movq %rax,lst+0x18(%rip) 119 leaq la57_trampoline_end(%rip),%rsp /* priv stack */ 120 121 movq %cr0,%rbp 122 leaq la57_trampoline_gdt(%rip),%rax 123 movq %rax,la57_trampoline_gdt_desc+2(%rip) 124 lgdtq la57_trampoline_gdt_desc(%rip) 125 126 pushq $(2<<3) 127 leaq l1(%rip),%rax 128 leaq l2(%rip),%rbx 129 130 pushq %rax 131 lretq 132 .code32 133 134l1: movl $(3<<3),%eax 135 movl %eax,%ss /* 32bit paged, priv gdt and stack */ 136 137 movl %cr4,%eax 138 andl $~(CR4_PGE | CR4_PCIDE),%eax /* clear sensitive paging ctrls */ 139 movl %eax,%cr4 140 141 movl %ebp,%eax 142 andl $~CR0_PG,%eax /* protected mode */ 143 movl %eax,%cr0 144 145 movl $MSR_EFER,%ecx /* disable long mode bit */ 146 rdmsr /* to safer tweaking LA57 */ 147 andl $~EFER_LME,%eax 148 wrmsr 149 150 movl %cr4,%eax /* finally safe to switch bit */ 151 orl $CR4_LA57,%eax 152 movl %eax,%cr4 153 154 movl %edi,%cr3 /* and load the 5-level pgtable root */ 155 156 rdmsr 157 orl $EFER_LME,%eax 158 wrmsr /* prepare for ... */ 159 160 movl %ebp,%cr0 /* and jump back directly into long */ 161 jmp 1f /* mode from protected by enabling pg */ 162 1631: pushl $(1<<3) /* reload %cs */ 164 pushl %ebx 165 lretl 166 .code64 167 168l2: movq lst(%rip),%rsp /* back on C stack */ 169 movq lst+8(%rip),%rbx 170 movq lst+0x10(%rip),%rbp 171 movq lst+0x18(%rip),%rax 172 movq %rax,%cr4 /* re-enable paging controls */ 173 retq /* back to C */ 174 .p2align 4,0 175lst: .quad 0,0,0,0 176ENTRY(la57_trampoline_gdt_desc) 177 .word la57_trampoline_end - la57_trampoline_gdt 178 .long 0, 0 /* filled by pmap_bootstrap_la57 */ 179 .p2align 4,0 180ENTRY(la57_trampoline_gdt) 181 .long 0x00000000 /* null desc */ 182 .long 0x00000000 183 .long 0x00000000 /* 64bit code */ 184 .long 0x00209800 185 .long 0x0000ffff /* 32bit code */ 186 .long 0x00cf9b00 187 .long 0x0000ffff /* universal data */ 188 .long 0x00cf9300 189 .dcb.l 16,0 190ENTRY(la57_trampoline_end) 191 192 .bss 193 .p2align PAGE_SHIFT 194 .globl bootstack 195 .space BOOTSTACK_SIZE /* space for bootstack - temporary stack */ 196bootstack: 197