1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/init.h> 8 #include <asm/fpu.h> 9 #include <asm/smp.h> 10 11 static unsigned int euen_mask = CSR_EUEN_FPEN; 12 13 /* 14 * The critical section between kernel_fpu_begin() and kernel_fpu_end() 15 * is non-reentrant. It is the caller's responsibility to avoid reentrance. 16 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example. 17 */ 18 static DEFINE_PER_CPU(bool, in_kernel_fpu); 19 static DEFINE_PER_CPU(unsigned int, euen_current); 20 fpregs_lock(void)21static inline void fpregs_lock(void) 22 { 23 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 24 preempt_disable(); 25 else 26 local_bh_disable(); 27 } 28 fpregs_unlock(void)29static inline void fpregs_unlock(void) 30 { 31 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 32 preempt_enable(); 33 else 34 local_bh_enable(); 35 } 36 kernel_fpu_begin(void)37void kernel_fpu_begin(void) 38 { 39 unsigned int *euen_curr; 40 41 if (!irqs_disabled()) 42 fpregs_lock(); 43 44 WARN_ON(this_cpu_read(in_kernel_fpu)); 45 46 this_cpu_write(in_kernel_fpu, true); 47 euen_curr = this_cpu_ptr(&euen_current); 48 49 *euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN); 50 51 #ifdef CONFIG_CPU_HAS_LASX 52 if (*euen_curr & CSR_EUEN_LASXEN) 53 _save_lasx(¤t->thread.fpu); 54 else 55 #endif 56 #ifdef CONFIG_CPU_HAS_LSX 57 if (*euen_curr & CSR_EUEN_LSXEN) 58 _save_lsx(¤t->thread.fpu); 59 else 60 #endif 61 if (*euen_curr & CSR_EUEN_FPEN) 62 _save_fp(¤t->thread.fpu); 63 64 write_fcsr(LOONGARCH_FCSR0, 0); 65 } 66 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 67 kernel_fpu_end(void)68void kernel_fpu_end(void) 69 { 70 unsigned int *euen_curr; 71 72 WARN_ON(!this_cpu_read(in_kernel_fpu)); 73 74 euen_curr = this_cpu_ptr(&euen_current); 75 76 #ifdef CONFIG_CPU_HAS_LASX 77 if (*euen_curr & CSR_EUEN_LASXEN) 78 _restore_lasx(¤t->thread.fpu); 79 else 80 #endif 81 #ifdef CONFIG_CPU_HAS_LSX 82 if (*euen_curr & CSR_EUEN_LSXEN) 83 _restore_lsx(¤t->thread.fpu); 84 else 85 #endif 86 if (*euen_curr & CSR_EUEN_FPEN) 87 _restore_fp(¤t->thread.fpu); 88 89 *euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN); 90 91 this_cpu_write(in_kernel_fpu, false); 92 93 if (!irqs_disabled()) 94 fpregs_unlock(); 95 } 96 EXPORT_SYMBOL_GPL(kernel_fpu_end); 97 init_euen_mask(void)98static int __init init_euen_mask(void) 99 { 100 if (cpu_has_lsx) 101 euen_mask |= CSR_EUEN_LSXEN; 102 103 if (cpu_has_lasx) 104 euen_mask |= CSR_EUEN_LASXEN; 105 106 return 0; 107 } 108 arch_initcall(init_euen_mask); 109