1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/vfp/vfpmodule.c 4 * 5 * Copyright (C) 2004 ARM Limited. 6 * Written by Deep Blue Solutions Limited. 7 */ 8 #include <linux/types.h> 9 #include <linux/cpu.h> 10 #include <linux/cpu_pm.h> 11 #include <linux/hardirq.h> 12 #include <linux/kernel.h> 13 #include <linux/notifier.h> 14 #include <linux/signal.h> 15 #include <linux/sched/signal.h> 16 #include <linux/smp.h> 17 #include <linux/init.h> 18 #include <linux/uaccess.h> 19 #include <linux/user.h> 20 #include <linux/export.h> 21 #include <linux/perf_event.h> 22 23 #include <asm/cp15.h> 24 #include <asm/cputype.h> 25 #include <asm/system_info.h> 26 #include <asm/thread_notify.h> 27 #include <asm/traps.h> 28 #include <asm/vfp.h> 29 #include <asm/neon.h> 30 31 #include "vfpinstr.h" 32 #include "vfp.h" 33 34 static bool have_vfp __ro_after_init; 35 36 /* 37 * Dual-use variable. 38 * Used in startup: set to non-zero if VFP checks fail 39 * After startup, holds VFP architecture 40 */ 41 static unsigned int VFP_arch; 42 43 #ifdef CONFIG_CPU_FEROCEON 44 extern unsigned int VFP_arch_feroceon __alias(VFP_arch); 45 #endif 46 47 /* 48 * The pointer to the vfpstate structure of the thread which currently 49 * owns the context held in the VFP hardware, or NULL if the hardware 50 * context is invalid. 51 * 52 * For UP, this is sufficient to tell which thread owns the VFP context. 53 * However, for SMP, we also need to check the CPU number stored in the 54 * saved state too to catch migrations. 55 */ 56 union vfp_state *vfp_current_hw_state[NR_CPUS]; 57 58 /* 59 * Claim ownership of the VFP unit. 60 * 61 * The caller may change VFP registers until vfp_state_release() is called. 62 * 63 * local_bh_disable() is used to disable preemption and to disable VFP 64 * processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is 65 * not sufficient because it only serializes soft interrupt related sections 66 * via a local lock, but stays preemptible. Disabling preemption is the right 67 * choice here as bottom half processing is always in thread context on RT 68 * kernels so it implicitly prevents bottom half processing as well. 69 */ 70 static void vfp_state_hold(void) 71 { 72 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 73 local_bh_disable(); 74 else 75 preempt_disable(); 76 } 77 78 static void vfp_state_release(void) 79 { 80 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 81 local_bh_enable(); 82 else 83 preempt_enable(); 84 } 85 86 /* 87 * Is 'thread's most up to date state stored in this CPUs hardware? 88 * Must be called from non-preemptible context. 89 */ 90 static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread) 91 { 92 #ifdef CONFIG_SMP 93 if (thread->vfpstate.hard.cpu != cpu) 94 return false; 95 #endif 96 return vfp_current_hw_state[cpu] == &thread->vfpstate; 97 } 98 99 /* 100 * Force a reload of the VFP context from the thread structure. We do 101 * this by ensuring that access to the VFP hardware is disabled, and 102 * clear vfp_current_hw_state. Must be called from non-preemptible context. 103 */ 104 static void vfp_force_reload(unsigned int cpu, struct thread_info *thread) 105 { 106 if (vfp_state_in_hw(cpu, thread)) { 107 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 108 vfp_current_hw_state[cpu] = NULL; 109 } 110 #ifdef CONFIG_SMP 111 thread->vfpstate.hard.cpu = NR_CPUS; 112 #endif 113 } 114 115 /* 116 * Per-thread VFP initialization. 117 */ 118 static void vfp_thread_flush(struct thread_info *thread) 119 { 120 union vfp_state *vfp = &thread->vfpstate; 121 unsigned int cpu; 122 123 /* 124 * Disable VFP to ensure we initialize it first. We must ensure 125 * that the modification of vfp_current_hw_state[] and hardware 126 * disable are done for the same CPU and without preemption. 127 * 128 * Do this first to ensure that preemption won't overwrite our 129 * state saving should access to the VFP be enabled at this point. 130 */ 131 cpu = get_cpu(); 132 if (vfp_current_hw_state[cpu] == vfp) 133 vfp_current_hw_state[cpu] = NULL; 134 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 135 put_cpu(); 136 137 memset(vfp, 0, sizeof(union vfp_state)); 138 139 vfp->hard.fpexc = FPEXC_EN; 140 vfp->hard.fpscr = FPSCR_ROUND_NEAREST; 141 #ifdef CONFIG_SMP 142 vfp->hard.cpu = NR_CPUS; 143 #endif 144 } 145 146 static void vfp_thread_exit(struct thread_info *thread) 147 { 148 /* release case: Per-thread VFP cleanup. */ 149 union vfp_state *vfp = &thread->vfpstate; 150 unsigned int cpu = get_cpu(); 151 152 if (vfp_current_hw_state[cpu] == vfp) 153 vfp_current_hw_state[cpu] = NULL; 154 put_cpu(); 155 } 156 157 static void vfp_thread_copy(struct thread_info *thread) 158 { 159 struct thread_info *parent = current_thread_info(); 160 161 vfp_sync_hwstate(parent); 162 thread->vfpstate = parent->vfpstate; 163 #ifdef CONFIG_SMP 164 thread->vfpstate.hard.cpu = NR_CPUS; 165 #endif 166 } 167 168 /* 169 * When this function is called with the following 'cmd's, the following 170 * is true while this function is being run: 171 * THREAD_NOFTIFY_SWTICH: 172 * - the previously running thread will not be scheduled onto another CPU. 173 * - the next thread to be run (v) will not be running on another CPU. 174 * - thread->cpu is the local CPU number 175 * - not preemptible as we're called in the middle of a thread switch 176 * THREAD_NOTIFY_FLUSH: 177 * - the thread (v) will be running on the local CPU, so 178 * v === current_thread_info() 179 * - thread->cpu is the local CPU number at the time it is accessed, 180 * but may change at any time. 181 * - we could be preempted if tree preempt rcu is enabled, so 182 * it is unsafe to use thread->cpu. 183 * THREAD_NOTIFY_EXIT 184 * - we could be preempted if tree preempt rcu is enabled, so 185 * it is unsafe to use thread->cpu. 186 */ 187 static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) 188 { 189 struct thread_info *thread = v; 190 u32 fpexc; 191 #ifdef CONFIG_SMP 192 unsigned int cpu; 193 #endif 194 195 switch (cmd) { 196 case THREAD_NOTIFY_SWITCH: 197 fpexc = fmrx(FPEXC); 198 199 #ifdef CONFIG_SMP 200 cpu = thread->cpu; 201 202 /* 203 * On SMP, if VFP is enabled, save the old state in 204 * case the thread migrates to a different CPU. The 205 * restoring is done lazily. 206 */ 207 if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) 208 vfp_save_state(vfp_current_hw_state[cpu], fpexc); 209 #endif 210 211 /* 212 * Always disable VFP so we can lazily save/restore the 213 * old state. 214 */ 215 fmxr(FPEXC, fpexc & ~FPEXC_EN); 216 break; 217 218 case THREAD_NOTIFY_FLUSH: 219 vfp_thread_flush(thread); 220 break; 221 222 case THREAD_NOTIFY_EXIT: 223 vfp_thread_exit(thread); 224 break; 225 226 case THREAD_NOTIFY_COPY: 227 vfp_thread_copy(thread); 228 break; 229 } 230 231 return NOTIFY_DONE; 232 } 233 234 static struct notifier_block vfp_notifier_block = { 235 .notifier_call = vfp_notifier, 236 }; 237 238 /* 239 * Raise a SIGFPE for the current process. 240 * sicode describes the signal being raised. 241 */ 242 static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs) 243 { 244 /* 245 * This is the same as NWFPE, because it's not clear what 246 * this is used for 247 */ 248 current->thread.error_code = 0; 249 current->thread.trap_no = 6; 250 251 send_sig_fault(SIGFPE, sicode, 252 (void __user *)(instruction_pointer(regs) - 4), 253 current); 254 } 255 256 static void vfp_panic(char *reason, u32 inst) 257 { 258 int i; 259 260 pr_err("VFP: Error: %s\n", reason); 261 pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n", 262 fmrx(FPEXC), fmrx(FPSCR), inst); 263 for (i = 0; i < 32; i += 2) 264 pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n", 265 i, vfp_get_float(i), i+1, vfp_get_float(i+1)); 266 } 267 268 /* 269 * Process bitmask of exception conditions. 270 */ 271 static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr) 272 { 273 int si_code = 0; 274 275 pr_debug("VFP: raising exceptions %08x\n", exceptions); 276 277 if (exceptions == VFP_EXCEPTION_ERROR) { 278 vfp_panic("unhandled bounce", inst); 279 return FPE_FLTINV; 280 } 281 282 /* 283 * If any of the status flags are set, update the FPSCR. 284 * Comparison instructions always return at least one of 285 * these flags set. 286 */ 287 if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V)) 288 fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V); 289 290 fpscr |= exceptions; 291 292 fmxr(FPSCR, fpscr); 293 294 #define RAISE(stat,en,sig) \ 295 if (exceptions & stat && fpscr & en) \ 296 si_code = sig; 297 298 /* 299 * These are arranged in priority order, least to highest. 300 */ 301 RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV); 302 RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES); 303 RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND); 304 RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF); 305 RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV); 306 307 return si_code; 308 } 309 310 /* 311 * Emulate a VFP instruction. 312 */ 313 static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) 314 { 315 u32 exceptions = VFP_EXCEPTION_ERROR; 316 317 pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr); 318 319 if (INST_CPRTDO(inst)) { 320 if (!INST_CPRT(inst)) { 321 /* 322 * CPDO 323 */ 324 if (vfp_single(inst)) { 325 exceptions = vfp_single_cpdo(inst, fpscr); 326 } else { 327 exceptions = vfp_double_cpdo(inst, fpscr); 328 } 329 } else { 330 /* 331 * A CPRT instruction can not appear in FPINST2, nor 332 * can it cause an exception. Therefore, we do not 333 * have to emulate it. 334 */ 335 } 336 } else { 337 /* 338 * A CPDT instruction can not appear in FPINST2, nor can 339 * it cause an exception. Therefore, we do not have to 340 * emulate it. 341 */ 342 } 343 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->ARM_pc); 344 return exceptions & ~VFP_NAN_FLAG; 345 } 346 347 /* 348 * Package up a bounce condition. 349 */ 350 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) 351 { 352 u32 fpscr, orig_fpscr, fpsid, exceptions; 353 int si_code2 = 0; 354 int si_code = 0; 355 356 pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc); 357 358 /* 359 * At this point, FPEXC can have the following configuration: 360 * 361 * EX DEX IXE 362 * 0 1 x - synchronous exception 363 * 1 x 0 - asynchronous exception 364 * 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later 365 * 0 0 1 - synchronous on VFP9 (non-standard subarch 1 366 * implementation), undefined otherwise 367 * 368 * Clear various bits and enable access to the VFP so we can 369 * handle the bounce. 370 */ 371 fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK)); 372 373 fpsid = fmrx(FPSID); 374 orig_fpscr = fpscr = fmrx(FPSCR); 375 376 /* 377 * Check for the special VFP subarch 1 and FPSCR.IXE bit case 378 */ 379 if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT) 380 && (fpscr & FPSCR_IXE)) { 381 /* 382 * Synchronous exception, emulate the trigger instruction 383 */ 384 goto emulate; 385 } 386 387 if (fpexc & FPEXC_EX) { 388 /* 389 * Asynchronous exception. The instruction is read from FPINST 390 * and the interrupted instruction has to be restarted. 391 */ 392 trigger = fmrx(FPINST); 393 regs->ARM_pc -= 4; 394 } else if (!(fpexc & FPEXC_DEX)) { 395 /* 396 * Illegal combination of bits. It can be caused by an 397 * unallocated VFP instruction but with FPSCR.IXE set and not 398 * on VFP subarch 1. 399 */ 400 si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr); 401 goto exit; 402 } 403 404 /* 405 * Modify fpscr to indicate the number of iterations remaining. 406 * If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates 407 * whether FPEXC.VECITR or FPSCR.LEN is used. 408 */ 409 if (fpexc & (FPEXC_EX | FPEXC_VV)) { 410 u32 len; 411 412 len = fpexc + (1 << FPEXC_LENGTH_BIT); 413 414 fpscr &= ~FPSCR_LENGTH_MASK; 415 fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT); 416 } 417 418 /* 419 * Handle the first FP instruction. We used to take note of the 420 * FPEXC bounce reason, but this appears to be unreliable. 421 * Emulate the bounced instruction instead. 422 */ 423 exceptions = vfp_emulate_instruction(trigger, fpscr, regs); 424 if (exceptions) 425 si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr); 426 427 /* 428 * If there isn't a second FP instruction, exit now. Note that 429 * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1. 430 */ 431 if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) 432 goto exit; 433 434 /* 435 * The barrier() here prevents fpinst2 being read 436 * before the condition above. 437 */ 438 barrier(); 439 trigger = fmrx(FPINST2); 440 441 emulate: 442 exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); 443 if (exceptions) 444 si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr); 445 exit: 446 vfp_state_release(); 447 if (si_code2) 448 vfp_raise_sigfpe(si_code2, regs); 449 if (si_code) 450 vfp_raise_sigfpe(si_code, regs); 451 } 452 453 static void vfp_enable(void *unused) 454 { 455 u32 access; 456 457 BUG_ON(preemptible()); 458 access = get_copro_access(); 459 460 /* 461 * Enable full access to VFP (cp10 and cp11) 462 */ 463 set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11)); 464 } 465 466 /* Called by platforms on which we want to disable VFP because it may not be 467 * present on all CPUs within a SMP complex. Needs to be called prior to 468 * vfp_init(). 469 */ 470 void __init vfp_disable(void) 471 { 472 if (VFP_arch) { 473 pr_debug("%s: should be called prior to vfp_init\n", __func__); 474 return; 475 } 476 VFP_arch = 1; 477 } 478 479 #ifdef CONFIG_CPU_PM 480 static int vfp_pm_suspend(void) 481 { 482 struct thread_info *ti = current_thread_info(); 483 u32 fpexc = fmrx(FPEXC); 484 485 /* if vfp is on, then save state for resumption */ 486 if (fpexc & FPEXC_EN) { 487 pr_debug("%s: saving vfp state\n", __func__); 488 vfp_save_state(&ti->vfpstate, fpexc); 489 490 /* disable, just in case */ 491 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 492 } else if (vfp_current_hw_state[ti->cpu]) { 493 #ifndef CONFIG_SMP 494 fmxr(FPEXC, fpexc | FPEXC_EN); 495 vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc); 496 fmxr(FPEXC, fpexc); 497 #endif 498 } 499 500 /* clear any information we had about last context state */ 501 vfp_current_hw_state[ti->cpu] = NULL; 502 503 return 0; 504 } 505 506 static void vfp_pm_resume(void) 507 { 508 /* ensure we have access to the vfp */ 509 vfp_enable(NULL); 510 511 /* and disable it to ensure the next usage restores the state */ 512 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 513 } 514 515 static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, 516 void *v) 517 { 518 switch (cmd) { 519 case CPU_PM_ENTER: 520 vfp_pm_suspend(); 521 break; 522 case CPU_PM_ENTER_FAILED: 523 case CPU_PM_EXIT: 524 vfp_pm_resume(); 525 break; 526 } 527 return NOTIFY_OK; 528 } 529 530 static struct notifier_block vfp_cpu_pm_notifier_block = { 531 .notifier_call = vfp_cpu_pm_notifier, 532 }; 533 534 static void vfp_pm_init(void) 535 { 536 cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block); 537 } 538 539 #else 540 static inline void vfp_pm_init(void) { } 541 #endif /* CONFIG_CPU_PM */ 542 543 /* 544 * Ensure that the VFP state stored in 'thread->vfpstate' is up to date 545 * with the hardware state. 546 */ 547 void vfp_sync_hwstate(struct thread_info *thread) 548 { 549 vfp_state_hold(); 550 551 if (vfp_state_in_hw(raw_smp_processor_id(), thread)) { 552 u32 fpexc = fmrx(FPEXC); 553 554 /* 555 * Save the last VFP state on this CPU. 556 */ 557 fmxr(FPEXC, fpexc | FPEXC_EN); 558 vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN); 559 fmxr(FPEXC, fpexc); 560 } 561 562 vfp_state_release(); 563 } 564 565 /* Ensure that the thread reloads the hardware VFP state on the next use. */ 566 void vfp_flush_hwstate(struct thread_info *thread) 567 { 568 unsigned int cpu = get_cpu(); 569 570 vfp_force_reload(cpu, thread); 571 572 put_cpu(); 573 } 574 575 /* 576 * Save the current VFP state into the provided structures and prepare 577 * for entry into a new function (signal handler). 578 */ 579 int vfp_preserve_user_clear_hwstate(struct user_vfp *ufp, 580 struct user_vfp_exc *ufp_exc) 581 { 582 struct thread_info *thread = current_thread_info(); 583 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; 584 585 /* Ensure that the saved hwstate is up-to-date. */ 586 vfp_sync_hwstate(thread); 587 588 /* 589 * Copy the floating point registers. There can be unused 590 * registers see asm/hwcap.h for details. 591 */ 592 memcpy(&ufp->fpregs, &hwstate->fpregs, sizeof(hwstate->fpregs)); 593 594 /* 595 * Copy the status and control register. 596 */ 597 ufp->fpscr = hwstate->fpscr; 598 599 /* 600 * Copy the exception registers. 601 */ 602 ufp_exc->fpexc = hwstate->fpexc; 603 ufp_exc->fpinst = hwstate->fpinst; 604 ufp_exc->fpinst2 = hwstate->fpinst2; 605 606 /* Ensure that VFP is disabled. */ 607 vfp_flush_hwstate(thread); 608 609 /* 610 * As per the PCS, clear the length and stride bits for function 611 * entry. 612 */ 613 hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK); 614 return 0; 615 } 616 617 /* Sanitise and restore the current VFP state from the provided structures. */ 618 int vfp_restore_user_hwstate(struct user_vfp *ufp, struct user_vfp_exc *ufp_exc) 619 { 620 struct thread_info *thread = current_thread_info(); 621 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; 622 unsigned long fpexc; 623 624 /* Disable VFP to avoid corrupting the new thread state. */ 625 vfp_flush_hwstate(thread); 626 627 /* 628 * Copy the floating point registers. There can be unused 629 * registers see asm/hwcap.h for details. 630 */ 631 memcpy(&hwstate->fpregs, &ufp->fpregs, sizeof(hwstate->fpregs)); 632 /* 633 * Copy the status and control register. 634 */ 635 hwstate->fpscr = ufp->fpscr; 636 637 /* 638 * Sanitise and restore the exception registers. 639 */ 640 fpexc = ufp_exc->fpexc; 641 642 /* Ensure the VFP is enabled. */ 643 fpexc |= FPEXC_EN; 644 645 /* Ensure FPINST2 is invalid and the exception flag is cleared. */ 646 fpexc &= ~(FPEXC_EX | FPEXC_FP2V); 647 hwstate->fpexc = fpexc; 648 649 hwstate->fpinst = ufp_exc->fpinst; 650 hwstate->fpinst2 = ufp_exc->fpinst2; 651 652 return 0; 653 } 654 655 /* 656 * VFP hardware can lose all context when a CPU goes offline. 657 * As we will be running in SMP mode with CPU hotplug, we will save the 658 * hardware state at every thread switch. We clear our held state when 659 * a CPU has been killed, indicating that the VFP hardware doesn't contain 660 * a threads VFP state. When a CPU starts up, we re-enable access to the 661 * VFP hardware. The callbacks below are called on the CPU which 662 * is being offlined/onlined. 663 */ 664 static int vfp_dying_cpu(unsigned int cpu) 665 { 666 vfp_current_hw_state[cpu] = NULL; 667 return 0; 668 } 669 670 static int vfp_starting_cpu(unsigned int unused) 671 { 672 vfp_enable(NULL); 673 return 0; 674 } 675 676 static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr) 677 { 678 /* 679 * If we reach this point, a floating point exception has been raised 680 * while running in kernel mode. If the NEON/VFP unit was enabled at the 681 * time, it means a VFP instruction has been issued that requires 682 * software assistance to complete, something which is not currently 683 * supported in kernel mode. 684 * If the NEON/VFP unit was disabled, and the location pointed to below 685 * is properly preceded by a call to kernel_neon_begin(), something has 686 * caused the task to be scheduled out and back in again. In this case, 687 * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should 688 * be helpful in localizing the problem. 689 */ 690 if (fmrx(FPEXC) & FPEXC_EN) 691 pr_crit("BUG: unsupported FP instruction in kernel mode\n"); 692 else 693 pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); 694 pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC)); 695 return 1; 696 } 697 698 /* 699 * vfp_support_entry - Handle VFP exception 700 * 701 * @regs: pt_regs structure holding the register state at exception entry 702 * @trigger: The opcode of the instruction that triggered the exception 703 * 704 * Returns 0 if the exception was handled, or an error code otherwise. 705 */ 706 static int vfp_support_entry(struct pt_regs *regs, u32 trigger) 707 { 708 struct thread_info *ti = current_thread_info(); 709 u32 fpexc; 710 711 if (unlikely(!have_vfp)) 712 return -ENODEV; 713 714 if (!user_mode(regs)) 715 return vfp_kmode_exception(regs, trigger); 716 717 vfp_state_hold(); 718 fpexc = fmrx(FPEXC); 719 720 /* 721 * If the VFP unit was not enabled yet, we have to check whether the 722 * VFP state in the CPU's registers is the most recent VFP state 723 * associated with the process. On UP systems, we don't save the VFP 724 * state eagerly on a context switch, so we may need to save the 725 * VFP state to memory first, as it may belong to another process. 726 */ 727 if (!(fpexc & FPEXC_EN)) { 728 /* 729 * Enable the VFP unit but mask the FP exception flag for the 730 * time being, so we can access all the registers. 731 */ 732 fpexc |= FPEXC_EN; 733 fmxr(FPEXC, fpexc & ~FPEXC_EX); 734 735 /* 736 * Check whether or not the VFP state in the CPU's registers is 737 * the most recent VFP state associated with this task. On SMP, 738 * migration may result in multiple CPUs holding VFP states 739 * that belong to the same task, but only the most recent one 740 * is valid. 741 */ 742 if (!vfp_state_in_hw(ti->cpu, ti)) { 743 if (!IS_ENABLED(CONFIG_SMP) && 744 vfp_current_hw_state[ti->cpu] != NULL) { 745 /* 746 * This CPU is currently holding the most 747 * recent VFP state associated with another 748 * task, and we must save that to memory first. 749 */ 750 vfp_save_state(vfp_current_hw_state[ti->cpu], 751 fpexc); 752 } 753 754 /* 755 * We can now proceed with loading the task's VFP state 756 * from memory into the CPU registers. 757 */ 758 fpexc = vfp_load_state(&ti->vfpstate); 759 vfp_current_hw_state[ti->cpu] = &ti->vfpstate; 760 #ifdef CONFIG_SMP 761 /* 762 * Record that this CPU is now the one holding the most 763 * recent VFP state of the task. 764 */ 765 ti->vfpstate.hard.cpu = ti->cpu; 766 #endif 767 } 768 769 if (fpexc & FPEXC_EX) 770 /* 771 * Might as well handle the pending exception before 772 * retrying branch out before setting an FPEXC that 773 * stops us reading stuff. 774 */ 775 goto bounce; 776 777 /* 778 * No FP exception is pending: just enable the VFP and 779 * replay the instruction that trapped. 780 */ 781 fmxr(FPEXC, fpexc); 782 vfp_state_release(); 783 } else { 784 /* Check for synchronous or asynchronous exceptions */ 785 if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) { 786 u32 fpscr = fmrx(FPSCR); 787 788 /* 789 * On some implementations of the VFP subarch 1, 790 * setting FPSCR.IXE causes all the CDP instructions to 791 * be bounced synchronously without setting the 792 * FPEXC.EX bit 793 */ 794 if (!(fpscr & FPSCR_IXE)) { 795 if (!(fpscr & FPSCR_LENGTH_MASK)) { 796 pr_debug("not VFP\n"); 797 vfp_state_release(); 798 return -ENOEXEC; 799 } 800 fpexc |= FPEXC_DEX; 801 } 802 } 803 bounce: regs->ARM_pc += 4; 804 /* VFP_bounce() will invoke vfp_state_release() */ 805 VFP_bounce(trigger, fpexc, regs); 806 } 807 808 return 0; 809 } 810 811 static struct undef_hook neon_support_hook[] = {{ 812 .instr_mask = 0xfe000000, 813 .instr_val = 0xf2000000, 814 .cpsr_mask = PSR_T_BIT, 815 .cpsr_val = 0, 816 .fn = vfp_support_entry, 817 }, { 818 .instr_mask = 0xff100000, 819 .instr_val = 0xf4000000, 820 .cpsr_mask = PSR_T_BIT, 821 .cpsr_val = 0, 822 .fn = vfp_support_entry, 823 }, { 824 .instr_mask = 0xef000000, 825 .instr_val = 0xef000000, 826 .cpsr_mask = PSR_T_BIT, 827 .cpsr_val = PSR_T_BIT, 828 .fn = vfp_support_entry, 829 }, { 830 .instr_mask = 0xff100000, 831 .instr_val = 0xf9000000, 832 .cpsr_mask = PSR_T_BIT, 833 .cpsr_val = PSR_T_BIT, 834 .fn = vfp_support_entry, 835 }, { 836 .instr_mask = 0xff000800, 837 .instr_val = 0xfc000800, 838 .cpsr_mask = 0, 839 .cpsr_val = 0, 840 .fn = vfp_support_entry, 841 }, { 842 .instr_mask = 0xff000800, 843 .instr_val = 0xfd000800, 844 .cpsr_mask = 0, 845 .cpsr_val = 0, 846 .fn = vfp_support_entry, 847 }, { 848 .instr_mask = 0xff000800, 849 .instr_val = 0xfe000800, 850 .cpsr_mask = 0, 851 .cpsr_val = 0, 852 .fn = vfp_support_entry, 853 }}; 854 855 static struct undef_hook vfp_support_hook = { 856 .instr_mask = 0x0c000e00, 857 .instr_val = 0x0c000a00, 858 .fn = vfp_support_entry, 859 }; 860 861 #ifdef CONFIG_KERNEL_MODE_NEON 862 863 /* 864 * Kernel-side NEON support functions 865 */ 866 void kernel_neon_begin(void) 867 { 868 struct thread_info *thread = current_thread_info(); 869 unsigned int cpu; 870 u32 fpexc; 871 872 vfp_state_hold(); 873 874 /* 875 * Kernel mode NEON is only allowed outside of hardirq context with 876 * preemption and softirq processing disabled. This will make sure that 877 * the kernel mode NEON register contents never need to be preserved. 878 */ 879 BUG_ON(in_hardirq()); 880 cpu = __smp_processor_id(); 881 882 fpexc = fmrx(FPEXC) | FPEXC_EN; 883 fmxr(FPEXC, fpexc); 884 885 /* 886 * Save the userland NEON/VFP state. Under UP, 887 * the owner could be a task other than 'current' 888 */ 889 if (vfp_state_in_hw(cpu, thread)) 890 vfp_save_state(&thread->vfpstate, fpexc); 891 #ifndef CONFIG_SMP 892 else if (vfp_current_hw_state[cpu] != NULL) 893 vfp_save_state(vfp_current_hw_state[cpu], fpexc); 894 #endif 895 vfp_current_hw_state[cpu] = NULL; 896 } 897 EXPORT_SYMBOL(kernel_neon_begin); 898 899 void kernel_neon_end(void) 900 { 901 /* Disable the NEON/VFP unit. */ 902 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 903 vfp_state_release(); 904 } 905 EXPORT_SYMBOL(kernel_neon_end); 906 907 #endif /* CONFIG_KERNEL_MODE_NEON */ 908 909 static int __init vfp_detect(struct pt_regs *regs, unsigned int instr) 910 { 911 VFP_arch = UINT_MAX; /* mark as not present */ 912 regs->ARM_pc += 4; 913 return 0; 914 } 915 916 static struct undef_hook vfp_detect_hook __initdata = { 917 .instr_mask = 0x0c000e00, 918 .instr_val = 0x0c000a00, 919 .cpsr_mask = MODE_MASK, 920 .cpsr_val = SVC_MODE, 921 .fn = vfp_detect, 922 }; 923 924 /* 925 * VFP support code initialisation. 926 */ 927 static int __init vfp_init(void) 928 { 929 unsigned int vfpsid; 930 unsigned int cpu_arch = cpu_architecture(); 931 unsigned int isar6; 932 933 /* 934 * Enable the access to the VFP on all online CPUs so the 935 * following test on FPSID will succeed. 936 */ 937 if (cpu_arch >= CPU_ARCH_ARMv6) 938 on_each_cpu(vfp_enable, NULL, 1); 939 940 /* 941 * First check that there is a VFP that we can use. 942 * The handler is already setup to just log calls, so 943 * we just need to read the VFPSID register. 944 */ 945 register_undef_hook(&vfp_detect_hook); 946 barrier(); 947 vfpsid = fmrx(FPSID); 948 barrier(); 949 unregister_undef_hook(&vfp_detect_hook); 950 951 pr_info("VFP support v0.3: "); 952 if (VFP_arch) { 953 pr_cont("not present\n"); 954 return 0; 955 /* Extract the architecture on CPUID scheme */ 956 } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) { 957 VFP_arch = vfpsid & FPSID_CPUID_ARCH_MASK; 958 VFP_arch >>= FPSID_ARCH_BIT; 959 /* 960 * Check for the presence of the Advanced SIMD 961 * load/store instructions, integer and single 962 * precision floating point operations. Only check 963 * for NEON if the hardware has the MVFR registers. 964 */ 965 if (IS_ENABLED(CONFIG_NEON) && 966 (fmrx(MVFR1) & 0x000fff00) == 0x00011100) { 967 elf_hwcap |= HWCAP_NEON; 968 for (int i = 0; i < ARRAY_SIZE(neon_support_hook); i++) 969 register_undef_hook(&neon_support_hook[i]); 970 } 971 972 if (IS_ENABLED(CONFIG_VFPv3)) { 973 u32 mvfr0 = fmrx(MVFR0); 974 if (((mvfr0 & MVFR0_DP_MASK) >> MVFR0_DP_BIT) == 0x2 || 975 ((mvfr0 & MVFR0_SP_MASK) >> MVFR0_SP_BIT) == 0x2) { 976 elf_hwcap |= HWCAP_VFPv3; 977 /* 978 * Check for VFPv3 D16 and VFPv4 D16. CPUs in 979 * this configuration only have 16 x 64bit 980 * registers. 981 */ 982 if ((mvfr0 & MVFR0_A_SIMD_MASK) == 1) 983 /* also v4-D16 */ 984 elf_hwcap |= HWCAP_VFPv3D16; 985 else 986 elf_hwcap |= HWCAP_VFPD32; 987 } 988 989 if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000) 990 elf_hwcap |= HWCAP_VFPv4; 991 if (((fmrx(MVFR1) & MVFR1_ASIMDHP_MASK) >> MVFR1_ASIMDHP_BIT) == 0x2) 992 elf_hwcap |= HWCAP_ASIMDHP; 993 if (((fmrx(MVFR1) & MVFR1_FPHP_MASK) >> MVFR1_FPHP_BIT) == 0x3) 994 elf_hwcap |= HWCAP_FPHP; 995 } 996 997 /* 998 * Check for the presence of Advanced SIMD Dot Product 999 * instructions. 1000 */ 1001 isar6 = read_cpuid_ext(CPUID_EXT_ISAR6); 1002 if (cpuid_feature_extract_field(isar6, 4) == 0x1) 1003 elf_hwcap |= HWCAP_ASIMDDP; 1004 /* 1005 * Check for the presence of Advanced SIMD Floating point 1006 * half-precision multiplication instructions. 1007 */ 1008 if (cpuid_feature_extract_field(isar6, 8) == 0x1) 1009 elf_hwcap |= HWCAP_ASIMDFHM; 1010 /* 1011 * Check for the presence of Advanced SIMD Bfloat16 1012 * floating point instructions. 1013 */ 1014 if (cpuid_feature_extract_field(isar6, 20) == 0x1) 1015 elf_hwcap |= HWCAP_ASIMDBF16; 1016 /* 1017 * Check for the presence of Advanced SIMD and floating point 1018 * Int8 matrix multiplication instructions instructions. 1019 */ 1020 if (cpuid_feature_extract_field(isar6, 24) == 0x1) 1021 elf_hwcap |= HWCAP_I8MM; 1022 1023 /* Extract the architecture version on pre-cpuid scheme */ 1024 } else { 1025 if (vfpsid & FPSID_NODOUBLE) { 1026 pr_cont("no double precision support\n"); 1027 return 0; 1028 } 1029 1030 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; 1031 } 1032 1033 cpuhp_setup_state_nocalls(CPUHP_AP_ARM_VFP_STARTING, 1034 "arm/vfp:starting", vfp_starting_cpu, 1035 vfp_dying_cpu); 1036 1037 have_vfp = true; 1038 1039 register_undef_hook(&vfp_support_hook); 1040 thread_register_notifier(&vfp_notifier_block); 1041 vfp_pm_init(); 1042 1043 /* 1044 * We detected VFP, and the support code is 1045 * in place; report VFP support to userspace. 1046 */ 1047 elf_hwcap |= HWCAP_VFP; 1048 1049 pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n", 1050 (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT, 1051 VFP_arch, 1052 (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT, 1053 (vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT, 1054 (vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT); 1055 1056 return 0; 1057 } 1058 1059 core_initcall(vfp_init); 1060