1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Pentium III FXSR, SSE support 6 * General FPU state handling cleanups 7 * Gareth Hughes <gareth@valinux.com>, May 2000 8 */ 9 #include <asm/fpu/internal.h> 10 #include <asm/fpu/regset.h> 11 #include <asm/fpu/signal.h> 12 #include <asm/fpu/types.h> 13 #include <asm/traps.h> 14 #include <asm/irq_regs.h> 15 16 #include <linux/hardirq.h> 17 #include <linux/pkeys.h> 18 19 #define CREATE_TRACE_POINTS 20 #include <asm/trace/fpu.h> 21 22 /* 23 * Represents the initial FPU state. It's mostly (but not completely) zeroes, 24 * depending on the FPU hardware format: 25 */ 26 union fpregs_state init_fpstate __read_mostly; 27 28 /* 29 * Track whether the kernel is using the FPU state 30 * currently. 31 * 32 * This flag is used: 33 * 34 * - by IRQ context code to potentially use the FPU 35 * if it's unused. 36 * 37 * - to debug kernel_fpu_begin()/end() correctness 38 */ 39 static DEFINE_PER_CPU(bool, in_kernel_fpu); 40 41 /* 42 * Track which context is using the FPU on the CPU: 43 */ 44 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); 45 46 static bool kernel_fpu_disabled(void) 47 { 48 return this_cpu_read(in_kernel_fpu); 49 } 50 51 static bool interrupted_kernel_fpu_idle(void) 52 { 53 return !kernel_fpu_disabled(); 54 } 55 56 /* 57 * Were we in user mode (or vm86 mode) when we were 58 * interrupted? 59 * 60 * Doing kernel_fpu_begin/end() is ok if we are running 61 * in an interrupt context from user mode - we'll just 62 * save the FPU state as required. 63 */ 64 static bool interrupted_user_mode(void) 65 { 66 struct pt_regs *regs = get_irq_regs(); 67 return regs && user_mode(regs); 68 } 69 70 /* 71 * Can we use the FPU in kernel mode with the 72 * whole "kernel_fpu_begin/end()" sequence? 73 * 74 * It's always ok in process context (ie "not interrupt") 75 * but it is sometimes ok even from an irq. 76 */ 77 bool irq_fpu_usable(void) 78 { 79 return !in_interrupt() || 80 interrupted_user_mode() || 81 interrupted_kernel_fpu_idle(); 82 } 83 EXPORT_SYMBOL(irq_fpu_usable); 84 85 void kernel_fpu_begin(void) 86 { 87 preempt_disable(); 88 89 WARN_ON_FPU(!irq_fpu_usable()); 90 WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); 91 92 this_cpu_write(in_kernel_fpu, true); 93 94 if (!(current->flags & PF_KTHREAD) && 95 !test_thread_flag(TIF_NEED_FPU_LOAD)) { 96 set_thread_flag(TIF_NEED_FPU_LOAD); 97 /* 98 * Ignore return value -- we don't care if reg state 99 * is clobbered. 100 */ 101 copy_fpregs_to_fpstate(¤t->thread.fpu); 102 } 103 __cpu_invalidate_fpregs_state(); 104 } 105 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 106 107 void kernel_fpu_end(void) 108 { 109 WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); 110 111 this_cpu_write(in_kernel_fpu, false); 112 preempt_enable(); 113 } 114 EXPORT_SYMBOL_GPL(kernel_fpu_end); 115 116 /* 117 * Save the FPU state (mark it for reload if necessary): 118 * 119 * This only ever gets called for the current task. 120 */ 121 void fpu__save(struct fpu *fpu) 122 { 123 WARN_ON_FPU(fpu != ¤t->thread.fpu); 124 125 fpregs_lock(); 126 trace_x86_fpu_before_save(fpu); 127 128 if (!test_thread_flag(TIF_NEED_FPU_LOAD)) { 129 if (!copy_fpregs_to_fpstate(fpu)) { 130 copy_kernel_to_fpregs(&fpu->state); 131 } 132 } 133 134 trace_x86_fpu_after_save(fpu); 135 fpregs_unlock(); 136 } 137 138 /* 139 * Legacy x87 fpstate state init: 140 */ 141 static inline void fpstate_init_fstate(struct fregs_state *fp) 142 { 143 fp->cwd = 0xffff037fu; 144 fp->swd = 0xffff0000u; 145 fp->twd = 0xffffffffu; 146 fp->fos = 0xffff0000u; 147 } 148 149 void fpstate_init(union fpregs_state *state) 150 { 151 if (!static_cpu_has(X86_FEATURE_FPU)) { 152 fpstate_init_soft(&state->soft); 153 return; 154 } 155 156 memset(state, 0, fpu_kernel_xstate_size); 157 158 if (static_cpu_has(X86_FEATURE_XSAVES)) 159 fpstate_init_xstate(&state->xsave); 160 if (static_cpu_has(X86_FEATURE_FXSR)) 161 fpstate_init_fxstate(&state->fxsave); 162 else 163 fpstate_init_fstate(&state->fsave); 164 } 165 EXPORT_SYMBOL_GPL(fpstate_init); 166 167 int fpu__copy(struct task_struct *dst, struct task_struct *src) 168 { 169 struct fpu *dst_fpu = &dst->thread.fpu; 170 struct fpu *src_fpu = &src->thread.fpu; 171 172 dst_fpu->last_cpu = -1; 173 174 if (!static_cpu_has(X86_FEATURE_FPU)) 175 return 0; 176 177 WARN_ON_FPU(src_fpu != ¤t->thread.fpu); 178 179 /* 180 * Don't let 'init optimized' areas of the XSAVE area 181 * leak into the child task: 182 */ 183 memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size); 184 185 /* 186 * If the FPU registers are not current just memcpy() the state. 187 * Otherwise save current FPU registers directly into the child's FPU 188 * context, without any memory-to-memory copying. 189 * 190 * ( The function 'fails' in the FNSAVE case, which destroys 191 * register contents so we have to load them back. ) 192 */ 193 fpregs_lock(); 194 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 195 memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size); 196 197 else if (!copy_fpregs_to_fpstate(dst_fpu)) 198 copy_kernel_to_fpregs(&dst_fpu->state); 199 200 fpregs_unlock(); 201 202 set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD); 203 204 trace_x86_fpu_copy_src(src_fpu); 205 trace_x86_fpu_copy_dst(dst_fpu); 206 207 return 0; 208 } 209 210 /* 211 * Activate the current task's in-memory FPU context, 212 * if it has not been used before: 213 */ 214 static void fpu__initialize(struct fpu *fpu) 215 { 216 WARN_ON_FPU(fpu != ¤t->thread.fpu); 217 218 set_thread_flag(TIF_NEED_FPU_LOAD); 219 fpstate_init(&fpu->state); 220 trace_x86_fpu_init_state(fpu); 221 } 222 223 /* 224 * This function must be called before we read a task's fpstate. 225 * 226 * There's two cases where this gets called: 227 * 228 * - for the current task (when coredumping), in which case we have 229 * to save the latest FPU registers into the fpstate, 230 * 231 * - or it's called for stopped tasks (ptrace), in which case the 232 * registers were already saved by the context-switch code when 233 * the task scheduled out. 234 * 235 * If the task has used the FPU before then save it. 236 */ 237 void fpu__prepare_read(struct fpu *fpu) 238 { 239 if (fpu == ¤t->thread.fpu) 240 fpu__save(fpu); 241 } 242 243 /* 244 * This function must be called before we write a task's fpstate. 245 * 246 * Invalidate any cached FPU registers. 247 * 248 * After this function call, after registers in the fpstate are 249 * modified and the child task has woken up, the child task will 250 * restore the modified FPU state from the modified context. If we 251 * didn't clear its cached status here then the cached in-registers 252 * state pending on its former CPU could be restored, corrupting 253 * the modifications. 254 */ 255 void fpu__prepare_write(struct fpu *fpu) 256 { 257 /* 258 * Only stopped child tasks can be used to modify the FPU 259 * state in the fpstate buffer: 260 */ 261 WARN_ON_FPU(fpu == ¤t->thread.fpu); 262 263 /* Invalidate any cached state: */ 264 __fpu_invalidate_fpregs_state(fpu); 265 } 266 267 /* 268 * Drops current FPU state: deactivates the fpregs and 269 * the fpstate. NOTE: it still leaves previous contents 270 * in the fpregs in the eager-FPU case. 271 * 272 * This function can be used in cases where we know that 273 * a state-restore is coming: either an explicit one, 274 * or a reschedule. 275 */ 276 void fpu__drop(struct fpu *fpu) 277 { 278 preempt_disable(); 279 280 if (fpu == ¤t->thread.fpu) { 281 /* Ignore delayed exceptions from user space */ 282 asm volatile("1: fwait\n" 283 "2:\n" 284 _ASM_EXTABLE(1b, 2b)); 285 fpregs_deactivate(fpu); 286 } 287 288 trace_x86_fpu_dropped(fpu); 289 290 preempt_enable(); 291 } 292 293 /* 294 * Clear FPU registers by setting them up from 295 * the init fpstate: 296 */ 297 static inline void copy_init_fpstate_to_fpregs(void) 298 { 299 fpregs_lock(); 300 301 if (use_xsave()) 302 copy_kernel_to_xregs(&init_fpstate.xsave, -1); 303 else if (static_cpu_has(X86_FEATURE_FXSR)) 304 copy_kernel_to_fxregs(&init_fpstate.fxsave); 305 else 306 copy_kernel_to_fregs(&init_fpstate.fsave); 307 308 if (boot_cpu_has(X86_FEATURE_OSPKE)) 309 copy_init_pkru_to_fpregs(); 310 311 fpregs_mark_activate(); 312 fpregs_unlock(); 313 } 314 315 /* 316 * Clear the FPU state back to init state. 317 * 318 * Called by sys_execve(), by the signal handler code and by various 319 * error paths. 320 */ 321 void fpu__clear(struct fpu *fpu) 322 { 323 WARN_ON_FPU(fpu != ¤t->thread.fpu); /* Almost certainly an anomaly */ 324 325 fpu__drop(fpu); 326 327 /* 328 * Make sure fpstate is cleared and initialized. 329 */ 330 fpu__initialize(fpu); 331 if (static_cpu_has(X86_FEATURE_FPU)) 332 copy_init_fpstate_to_fpregs(); 333 } 334 335 /* 336 * Load FPU context before returning to userspace. 337 */ 338 void switch_fpu_return(void) 339 { 340 if (!static_cpu_has(X86_FEATURE_FPU)) 341 return; 342 343 __fpregs_load_activate(); 344 } 345 EXPORT_SYMBOL_GPL(switch_fpu_return); 346 347 #ifdef CONFIG_X86_DEBUG_FPU 348 /* 349 * If current FPU state according to its tracking (loaded FPU context on this 350 * CPU) is not valid then we must have TIF_NEED_FPU_LOAD set so the context is 351 * loaded on return to userland. 352 */ 353 void fpregs_assert_state_consistent(void) 354 { 355 struct fpu *fpu = ¤t->thread.fpu; 356 357 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 358 return; 359 360 WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id())); 361 } 362 EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); 363 #endif 364 365 void fpregs_mark_activate(void) 366 { 367 struct fpu *fpu = ¤t->thread.fpu; 368 369 fpregs_activate(fpu); 370 fpu->last_cpu = smp_processor_id(); 371 clear_thread_flag(TIF_NEED_FPU_LOAD); 372 } 373 EXPORT_SYMBOL_GPL(fpregs_mark_activate); 374 375 /* 376 * x87 math exception handling: 377 */ 378 379 int fpu__exception_code(struct fpu *fpu, int trap_nr) 380 { 381 int err; 382 383 if (trap_nr == X86_TRAP_MF) { 384 unsigned short cwd, swd; 385 /* 386 * (~cwd & swd) will mask out exceptions that are not set to unmasked 387 * status. 0x3f is the exception bits in these regs, 0x200 is the 388 * C1 reg you need in case of a stack fault, 0x040 is the stack 389 * fault bit. We should only be taking one exception at a time, 390 * so if this combination doesn't produce any single exception, 391 * then we have a bad program that isn't synchronizing its FPU usage 392 * and it will suffer the consequences since we won't be able to 393 * fully reproduce the context of the exception. 394 */ 395 if (boot_cpu_has(X86_FEATURE_FXSR)) { 396 cwd = fpu->state.fxsave.cwd; 397 swd = fpu->state.fxsave.swd; 398 } else { 399 cwd = (unsigned short)fpu->state.fsave.cwd; 400 swd = (unsigned short)fpu->state.fsave.swd; 401 } 402 403 err = swd & ~cwd; 404 } else { 405 /* 406 * The SIMD FPU exceptions are handled a little differently, as there 407 * is only a single status/control register. Thus, to determine which 408 * unmasked exception was caught we must mask the exception mask bits 409 * at 0x1f80, and then use these to mask the exception bits at 0x3f. 410 */ 411 unsigned short mxcsr = MXCSR_DEFAULT; 412 413 if (boot_cpu_has(X86_FEATURE_XMM)) 414 mxcsr = fpu->state.fxsave.mxcsr; 415 416 err = ~(mxcsr >> 7) & mxcsr; 417 } 418 419 if (err & 0x001) { /* Invalid op */ 420 /* 421 * swd & 0x240 == 0x040: Stack Underflow 422 * swd & 0x240 == 0x240: Stack Overflow 423 * User must clear the SF bit (0x40) if set 424 */ 425 return FPE_FLTINV; 426 } else if (err & 0x004) { /* Divide by Zero */ 427 return FPE_FLTDIV; 428 } else if (err & 0x008) { /* Overflow */ 429 return FPE_FLTOVF; 430 } else if (err & 0x012) { /* Denormal, Underflow */ 431 return FPE_FLTUND; 432 } else if (err & 0x020) { /* Precision */ 433 return FPE_FLTRES; 434 } 435 436 /* 437 * If we're using IRQ 13, or supposedly even some trap 438 * X86_TRAP_MF implementations, it's possible 439 * we get a spurious trap, which is not an error. 440 */ 441 return 0; 442 } 443