1 #include <asm/processor.h> 2 #include "pgtable.h" 3 #include "../string.h" 4 5 /* 6 * __force_order is used by special_insns.h asm code to force instruction 7 * serialization. 8 * 9 * It is not referenced from the code, but GCC < 5 with -fPIE would fail 10 * due to an undefined symbol. Define it to make these ancient GCCs work. 11 */ 12 unsigned long __force_order; 13 14 #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ 15 #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ 16 17 struct paging_config { 18 unsigned long trampoline_start; 19 unsigned long l5_required; 20 }; 21 22 /* Buffer to preserve trampoline memory */ 23 static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; 24 25 /* 26 * Trampoline address will be printed by extract_kernel() for debugging 27 * purposes. 28 * 29 * Avoid putting the pointer into .bss as it will be cleared between 30 * paging_prepare() and extract_kernel(). 31 */ 32 unsigned long *trampoline_32bit __section(.data); 33 34 extern struct boot_params *boot_params; 35 int cmdline_find_option_bool(const char *option); 36 37 struct paging_config paging_prepare(void *rmode) 38 { 39 struct paging_config paging_config = {}; 40 unsigned long bios_start, ebda_start; 41 42 /* Initialize boot_params. Required for cmdline_find_option_bool(). */ 43 boot_params = rmode; 44 45 /* 46 * Check if LA57 is desired and supported. 47 * 48 * There are several parts to the check: 49 * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y 50 * - if user asked to disable 5-level paging: no5lvl in cmdline 51 * - if the machine supports 5-level paging: 52 * + CPUID leaf 7 is supported 53 * + the leaf has the feature bit set 54 * 55 * That's substitute for boot_cpu_has() in early boot code. 56 */ 57 if (IS_ENABLED(CONFIG_X86_5LEVEL) && 58 !cmdline_find_option_bool("no5lvl") && 59 native_cpuid_eax(0) >= 7 && 60 (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) { 61 paging_config.l5_required = 1; 62 } 63 64 /* 65 * Find a suitable spot for the trampoline. 66 * This code is based on reserve_bios_regions(). 67 */ 68 69 ebda_start = *(unsigned short *)0x40e << 4; 70 bios_start = *(unsigned short *)0x413 << 10; 71 72 if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) 73 bios_start = BIOS_START_MAX; 74 75 if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) 76 bios_start = ebda_start; 77 78 /* Place the trampoline just below the end of low memory, aligned to 4k */ 79 paging_config.trampoline_start = bios_start - TRAMPOLINE_32BIT_SIZE; 80 paging_config.trampoline_start = round_down(paging_config.trampoline_start, PAGE_SIZE); 81 82 trampoline_32bit = (unsigned long *)paging_config.trampoline_start; 83 84 /* Preserve trampoline memory */ 85 memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE); 86 87 /* Clear trampoline memory first */ 88 memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE); 89 90 /* Copy trampoline code in place */ 91 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long), 92 &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE); 93 94 /* 95 * The code below prepares page table in trampoline memory. 96 * 97 * The new page table will be used by trampoline code for switching 98 * from 4- to 5-level paging or vice versa. 99 * 100 * If switching is not required, the page table is unused: trampoline 101 * code wouldn't touch CR3. 102 */ 103 104 /* 105 * We are not going to use the page table in trampoline memory if we 106 * are already in the desired paging mode. 107 */ 108 if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57)) 109 goto out; 110 111 if (paging_config.l5_required) { 112 /* 113 * For 4- to 5-level paging transition, set up current CR3 as 114 * the first and the only entry in a new top-level page table. 115 */ 116 trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC; 117 } else { 118 unsigned long src; 119 120 /* 121 * For 5- to 4-level paging transition, copy page table pointed 122 * by first entry in the current top-level page table as our 123 * new top-level page table. 124 * 125 * We cannot just point to the page table from trampoline as it 126 * may be above 4G. 127 */ 128 src = *(unsigned long *)__native_read_cr3() & PAGE_MASK; 129 memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long), 130 (void *)src, PAGE_SIZE); 131 } 132 133 out: 134 return paging_config; 135 } 136 137 void cleanup_trampoline(void *pgtable) 138 { 139 void *trampoline_pgtable; 140 141 trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long); 142 143 /* 144 * Move the top level page table out of trampoline memory, 145 * if it's there. 146 */ 147 if ((void *)__native_read_cr3() == trampoline_pgtable) { 148 memcpy(pgtable, trampoline_pgtable, PAGE_SIZE); 149 native_write_cr3((unsigned long)pgtable); 150 } 151 152 /* Restore trampoline memory */ 153 memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); 154 } 155