1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * x86 FPU boot time init code: 4 */ 5 #include <asm/fpu/api.h> 6 #include <asm/tlbflush.h> 7 #include <asm/setup.h> 8 9 #include <linux/sched.h> 10 #include <linux/sched/task.h> 11 #include <linux/init.h> 12 13 #include "internal.h" 14 #include "legacy.h" 15 #include "xstate.h" 16 17 /* 18 * Initialize the registers found in all CPUs, CR0 and CR4: 19 */ 20 static void fpu__init_cpu_generic(void) 21 { 22 unsigned long cr0; 23 unsigned long cr4_mask = 0; 24 25 if (boot_cpu_has(X86_FEATURE_FXSR)) 26 cr4_mask |= X86_CR4_OSFXSR; 27 if (boot_cpu_has(X86_FEATURE_XMM)) 28 cr4_mask |= X86_CR4_OSXMMEXCPT; 29 if (cr4_mask) 30 cr4_set_bits(cr4_mask); 31 32 cr0 = read_cr0(); 33 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */ 34 write_cr0(cr0); 35 36 /* Flush out any pending x87 state: */ 37 asm volatile ("fninit"); 38 } 39 40 /* 41 * Enable all supported FPU features. Called when a CPU is brought online: 42 */ 43 void fpu__init_cpu(void) 44 { 45 fpu__init_cpu_generic(); 46 fpu__init_cpu_xstate(); 47 48 /* Start allowing kernel-mode FPU: */ 49 this_cpu_write(kernel_fpu_allowed, true); 50 } 51 52 static bool __init fpu__probe_without_cpuid(void) 53 { 54 unsigned long cr0; 55 u16 fsw, fcw; 56 57 fsw = fcw = 0xffff; 58 59 cr0 = read_cr0(); 60 cr0 &= ~(X86_CR0_TS | X86_CR0_EM); 61 write_cr0(cr0); 62 63 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw)); 64 65 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw); 66 67 return fsw == 0 && (fcw & 0x103f) == 0x003f; 68 } 69 70 static void __init fpu__init_system_early_generic(void) 71 { 72 set_thread_flag(TIF_NEED_FPU_LOAD); 73 74 if (!boot_cpu_has(X86_FEATURE_CPUID) && 75 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { 76 if (fpu__probe_without_cpuid()) 77 setup_force_cpu_cap(X86_FEATURE_FPU); 78 else 79 setup_clear_cpu_cap(X86_FEATURE_FPU); 80 } 81 82 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) { 83 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n"); 84 for (;;) 85 asm volatile("hlt"); 86 } 87 } 88 89 /* 90 * Boot time FPU feature detection code: 91 */ 92 unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu; 93 94 static void __init fpu__init_system_mxcsr(void) 95 { 96 unsigned int mask = 0; 97 98 if (boot_cpu_has(X86_FEATURE_FXSR)) { 99 /* Static because GCC does not get 16-byte stack alignment right: */ 100 static struct fxregs_state fxregs __initdata; 101 102 asm volatile("fxsave %0" : "+m" (fxregs)); 103 104 mask = fxregs.mxcsr_mask; 105 106 /* 107 * If zero then use the default features mask, 108 * which has all features set, except the 109 * denormals-are-zero feature bit: 110 */ 111 if (mask == 0) 112 mask = 0x0000ffbf; 113 } 114 mxcsr_feature_mask &= mask; 115 } 116 117 /* 118 * Once per bootup FPU initialization sequences that will run on most x86 CPUs: 119 */ 120 static void __init fpu__init_system_generic(void) 121 { 122 /* 123 * Set up the legacy init FPU context. Will be updated when the 124 * CPU supports XSAVE[S]. 125 */ 126 fpstate_init_user(&init_fpstate); 127 128 fpu__init_system_mxcsr(); 129 } 130 131 /* 132 * Enforce that 'MEMBER' is the last field of 'TYPE'. 133 * 134 * Align the computed size with alignment of the TYPE, 135 * because that's how C aligns structs. 136 */ 137 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \ 138 BUILD_BUG_ON(sizeof(TYPE) != \ 139 ALIGN(offsetofend(TYPE, MEMBER), _Alignof(TYPE))) 140 141 /* 142 * We append the 'struct fpu' to the task_struct: 143 */ 144 static void __init fpu__init_task_struct_size(void) 145 { 146 int task_size = sizeof(struct task_struct); 147 148 task_size += sizeof(struct fpu); 149 150 /* 151 * Subtract off the static size of the register state. 152 * It potentially has a bunch of padding. 153 */ 154 task_size -= sizeof(union fpregs_state); 155 156 /* 157 * Add back the dynamically-calculated register state 158 * size. 159 */ 160 task_size += fpu_kernel_cfg.default_size; 161 162 /* 163 * We dynamically size 'struct fpu', so we require that 164 * 'state' be at the end of 'it: 165 */ 166 CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate); 167 168 arch_task_struct_size = task_size; 169 } 170 171 /* 172 * Set up the user and kernel xstate sizes based on the legacy FPU context size. 173 * 174 * We set this up first, and later it will be overwritten by 175 * fpu__init_system_xstate() if the CPU knows about xstates. 176 */ 177 static void __init fpu__init_system_xstate_size_legacy(void) 178 { 179 unsigned int size; 180 181 /* 182 * Note that the size configuration might be overwritten later 183 * during fpu__init_system_xstate(). 184 */ 185 if (cpu_feature_enabled(X86_FEATURE_FXSR)) { 186 size = sizeof(struct fxregs_state); 187 fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE; 188 } else { 189 size = sizeof(struct fregs_state); 190 fpu_user_cfg.legacy_features = XFEATURE_MASK_FP; 191 } 192 193 fpu_kernel_cfg.max_size = size; 194 fpu_kernel_cfg.default_size = size; 195 fpu_user_cfg.max_size = size; 196 fpu_user_cfg.default_size = size; 197 guest_default_cfg.size = size; 198 } 199 200 /* 201 * Called on the boot CPU once per system bootup, to set up the initial 202 * FPU state that is later cloned into all processes: 203 */ 204 void __init fpu__init_system(void) 205 { 206 fpu__init_system_early_generic(); 207 208 /* 209 * The FPU has to be operational for some of the 210 * later FPU init activities: 211 */ 212 fpu__init_cpu(); 213 214 fpu__init_system_generic(); 215 fpu__init_system_xstate_size_legacy(); 216 fpu__init_system_xstate(fpu_kernel_cfg.max_size); 217 fpu__init_task_struct_size(); 218 } 219