1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * x86 FPU boot time init code:
4 */
5 #include <asm/fpu/api.h>
6 #include <asm/tlbflush.h>
7 #include <asm/setup.h>
8
9 #include <linux/sched.h>
10 #include <linux/sched/task.h>
11 #include <linux/init.h>
12
13 #include "internal.h"
14 #include "legacy.h"
15 #include "xstate.h"
16
17 /*
18 * Initialize the registers found in all CPUs, CR0 and CR4:
19 */
fpu__init_cpu_generic(void)20 static void fpu__init_cpu_generic(void)
21 {
22 unsigned long cr0;
23 unsigned long cr4_mask = 0;
24
25 if (boot_cpu_has(X86_FEATURE_FXSR))
26 cr4_mask |= X86_CR4_OSFXSR;
27 if (boot_cpu_has(X86_FEATURE_XMM))
28 cr4_mask |= X86_CR4_OSXMMEXCPT;
29 if (cr4_mask)
30 cr4_set_bits(cr4_mask);
31
32 cr0 = read_cr0();
33 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
34 write_cr0(cr0);
35
36 /* Flush out any pending x87 state: */
37 asm volatile ("fninit");
38 }
39
40 /*
41 * Enable all supported FPU features. Called when a CPU is brought online:
42 */
fpu__init_cpu(void)43 void fpu__init_cpu(void)
44 {
45 fpu__init_cpu_generic();
46 fpu__init_cpu_xstate();
47
48 /* Start allowing kernel-mode FPU: */
49 this_cpu_write(kernel_fpu_allowed, true);
50 }
51
fpu__probe_without_cpuid(void)52 static bool __init fpu__probe_without_cpuid(void)
53 {
54 unsigned long cr0;
55 u16 fsw, fcw;
56
57 fsw = fcw = 0xffff;
58
59 cr0 = read_cr0();
60 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
61 write_cr0(cr0);
62
63 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
64
65 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
66
67 return fsw == 0 && (fcw & 0x103f) == 0x003f;
68 }
69
fpu__init_system_early_generic(void)70 static void __init fpu__init_system_early_generic(void)
71 {
72 set_thread_flag(TIF_NEED_FPU_LOAD);
73
74 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
75 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
76 if (fpu__probe_without_cpuid())
77 setup_force_cpu_cap(X86_FEATURE_FPU);
78 else
79 setup_clear_cpu_cap(X86_FEATURE_FPU);
80 }
81
82 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
83 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
84 for (;;)
85 asm volatile("hlt");
86 }
87 }
88
89 /*
90 * Boot time FPU feature detection code:
91 */
92 unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu;
93
fpu__init_system_mxcsr(void)94 static void __init fpu__init_system_mxcsr(void)
95 {
96 unsigned int mask = 0;
97
98 if (boot_cpu_has(X86_FEATURE_FXSR)) {
99 /* Static because GCC does not get 16-byte stack alignment right: */
100 static struct fxregs_state fxregs __initdata;
101
102 asm volatile("fxsave %0" : "+m" (fxregs));
103
104 mask = fxregs.mxcsr_mask;
105
106 /*
107 * If zero then use the default features mask,
108 * which has all features set, except the
109 * denormals-are-zero feature bit:
110 */
111 if (mask == 0)
112 mask = 0x0000ffbf;
113 }
114 mxcsr_feature_mask &= mask;
115 }
116
117 /*
118 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
119 */
fpu__init_system_generic(void)120 static void __init fpu__init_system_generic(void)
121 {
122 /*
123 * Set up the legacy init FPU context. Will be updated when the
124 * CPU supports XSAVE[S].
125 */
126 fpstate_init_user(&init_fpstate);
127
128 fpu__init_system_mxcsr();
129 }
130
131 /*
132 * Enforce that 'MEMBER' is the last field of 'TYPE'.
133 *
134 * Align the computed size with alignment of the TYPE,
135 * because that's how C aligns structs.
136 */
137 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
138 BUILD_BUG_ON(sizeof(TYPE) != \
139 ALIGN(offsetofend(TYPE, MEMBER), _Alignof(TYPE)))
140
141 /*
142 * We append the 'struct fpu' to the task_struct:
143 */
fpu__init_task_struct_size(void)144 static void __init fpu__init_task_struct_size(void)
145 {
146 int task_size = sizeof(struct task_struct);
147
148 task_size += sizeof(struct fpu);
149
150 /*
151 * Subtract off the static size of the register state.
152 * It potentially has a bunch of padding.
153 */
154 task_size -= sizeof(union fpregs_state);
155
156 /*
157 * Add back the dynamically-calculated register state
158 * size.
159 */
160 task_size += fpu_kernel_cfg.default_size;
161
162 /*
163 * We dynamically size 'struct fpu', so we require that
164 * 'state' be at the end of 'it:
165 */
166 CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);
167
168 arch_task_struct_size = task_size;
169 }
170
171 /*
172 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
173 *
174 * We set this up first, and later it will be overwritten by
175 * fpu__init_system_xstate() if the CPU knows about xstates.
176 */
fpu__init_system_xstate_size_legacy(void)177 static void __init fpu__init_system_xstate_size_legacy(void)
178 {
179 unsigned int size;
180
181 /*
182 * Note that the size configuration might be overwritten later
183 * during fpu__init_system_xstate().
184 */
185 if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
186 size = sizeof(struct fxregs_state);
187 fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE;
188 } else {
189 size = sizeof(struct fregs_state);
190 fpu_user_cfg.legacy_features = XFEATURE_MASK_FP;
191 }
192
193 fpu_kernel_cfg.max_size = size;
194 fpu_kernel_cfg.default_size = size;
195 fpu_user_cfg.max_size = size;
196 fpu_user_cfg.default_size = size;
197 guest_default_cfg.size = size;
198 }
199
200 /*
201 * Called on the boot CPU once per system bootup, to set up the initial
202 * FPU state that is later cloned into all processes:
203 */
fpu__init_system(void)204 void __init fpu__init_system(void)
205 {
206 fpu__init_system_early_generic();
207
208 /*
209 * The FPU has to be operational for some of the
210 * later FPU init activities:
211 */
212 fpu__init_cpu();
213
214 fpu__init_system_generic();
215 fpu__init_system_xstate_size_legacy();
216 fpu__init_system_xstate(fpu_kernel_cfg.max_size);
217 fpu__init_task_struct_size();
218 }
219