1 #include <asm/e820/types.h> 2 #include <asm/processor.h> 3 #include "pgtable.h" 4 #include "../string.h" 5 6 /* 7 * __force_order is used by special_insns.h asm code to force instruction 8 * serialization. 9 * 10 * It is not referenced from the code, but GCC < 5 with -fPIE would fail 11 * due to an undefined symbol. Define it to make these ancient GCCs work. 12 */ 13 unsigned long __force_order; 14 15 #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ 16 #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ 17 18 struct paging_config { 19 unsigned long trampoline_start; 20 unsigned long l5_required; 21 }; 22 23 /* Buffer to preserve trampoline memory */ 24 static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; 25 26 /* 27 * Trampoline address will be printed by extract_kernel() for debugging 28 * purposes. 29 * 30 * Avoid putting the pointer into .bss as it will be cleared between 31 * paging_prepare() and extract_kernel(). 32 */ 33 unsigned long *trampoline_32bit __section(.data); 34 35 extern struct boot_params *boot_params; 36 int cmdline_find_option_bool(const char *option); 37 38 static unsigned long find_trampoline_placement(void) 39 { 40 unsigned long bios_start, ebda_start; 41 unsigned long trampoline_start; 42 struct boot_e820_entry *entry; 43 int i; 44 45 /* 46 * Find a suitable spot for the trampoline. 47 * This code is based on reserve_bios_regions(). 48 */ 49 50 ebda_start = *(unsigned short *)0x40e << 4; 51 bios_start = *(unsigned short *)0x413 << 10; 52 53 if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) 54 bios_start = BIOS_START_MAX; 55 56 if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) 57 bios_start = ebda_start; 58 59 bios_start = round_down(bios_start, PAGE_SIZE); 60 61 /* Find the first usable memory region under bios_start. */ 62 for (i = boot_params->e820_entries - 1; i >= 0; i--) { 63 entry = &boot_params->e820_table[i]; 64 65 /* Skip all entries above bios_start. */ 66 if (bios_start <= entry->addr) 67 continue; 68 69 /* Skip non-RAM entries. */ 70 if (entry->type != E820_TYPE_RAM) 71 continue; 72 73 /* Adjust bios_start to the end of the entry if needed. */ 74 if (bios_start > entry->addr + entry->size) 75 bios_start = entry->addr + entry->size; 76 77 /* Keep bios_start page-aligned. */ 78 bios_start = round_down(bios_start, PAGE_SIZE); 79 80 /* Skip the entry if it's too small. */ 81 if (bios_start - TRAMPOLINE_32BIT_SIZE < entry->addr) 82 continue; 83 84 break; 85 } 86 87 /* Place the trampoline just below the end of low memory */ 88 return bios_start - TRAMPOLINE_32BIT_SIZE; 89 } 90 91 struct paging_config paging_prepare(void *rmode) 92 { 93 struct paging_config paging_config = {}; 94 95 /* Initialize boot_params. Required for cmdline_find_option_bool(). */ 96 boot_params = rmode; 97 98 /* 99 * Check if LA57 is desired and supported. 100 * 101 * There are several parts to the check: 102 * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y 103 * - if user asked to disable 5-level paging: no5lvl in cmdline 104 * - if the machine supports 5-level paging: 105 * + CPUID leaf 7 is supported 106 * + the leaf has the feature bit set 107 * 108 * That's substitute for boot_cpu_has() in early boot code. 109 */ 110 if (IS_ENABLED(CONFIG_X86_5LEVEL) && 111 !cmdline_find_option_bool("no5lvl") && 112 native_cpuid_eax(0) >= 7 && 113 (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) { 114 paging_config.l5_required = 1; 115 } 116 117 paging_config.trampoline_start = find_trampoline_placement(); 118 119 trampoline_32bit = (unsigned long *)paging_config.trampoline_start; 120 121 /* Preserve trampoline memory */ 122 memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE); 123 124 /* Clear trampoline memory first */ 125 memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE); 126 127 /* Copy trampoline code in place */ 128 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long), 129 &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE); 130 131 /* 132 * The code below prepares page table in trampoline memory. 133 * 134 * The new page table will be used by trampoline code for switching 135 * from 4- to 5-level paging or vice versa. 136 * 137 * If switching is not required, the page table is unused: trampoline 138 * code wouldn't touch CR3. 139 */ 140 141 /* 142 * We are not going to use the page table in trampoline memory if we 143 * are already in the desired paging mode. 144 */ 145 if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57)) 146 goto out; 147 148 if (paging_config.l5_required) { 149 /* 150 * For 4- to 5-level paging transition, set up current CR3 as 151 * the first and the only entry in a new top-level page table. 152 */ 153 trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC; 154 } else { 155 unsigned long src; 156 157 /* 158 * For 5- to 4-level paging transition, copy page table pointed 159 * by first entry in the current top-level page table as our 160 * new top-level page table. 161 * 162 * We cannot just point to the page table from trampoline as it 163 * may be above 4G. 164 */ 165 src = *(unsigned long *)__native_read_cr3() & PAGE_MASK; 166 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long), 167 (void *)src, PAGE_SIZE); 168 } 169 170 out: 171 return paging_config; 172 } 173 174 void cleanup_trampoline(void *pgtable) 175 { 176 void *trampoline_pgtable; 177 178 trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long); 179 180 /* 181 * Move the top level page table out of trampoline memory, 182 * if it's there. 183 */ 184 if ((void *)__native_read_cr3() == trampoline_pgtable) { 185 memcpy(pgtable, trampoline_pgtable, PAGE_SIZE); 186 native_write_cr3((unsigned long)pgtable); 187 } 188 189 /* Restore trampoline memory */ 190 memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); 191 } 192