1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/export.h> 8 #include <linux/init.h> 9 #include <asm/fpu.h> 10 #include <asm/smp.h> 11 12 static unsigned int euen_mask = CSR_EUEN_FPEN; 13 14 /* 15 * The critical section between kernel_fpu_begin() and kernel_fpu_end() 16 * is non-reentrant. It is the caller's responsibility to avoid reentrance. 17 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example. 18 */ 19 static DEFINE_PER_CPU(bool, in_kernel_fpu); 20 static DEFINE_PER_CPU(unsigned int, euen_current); 21 fpregs_lock(void)22static inline void fpregs_lock(void) 23 { 24 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 25 preempt_disable(); 26 else 27 local_bh_disable(); 28 } 29 fpregs_unlock(void)30static inline void fpregs_unlock(void) 31 { 32 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 33 preempt_enable(); 34 else 35 local_bh_enable(); 36 } 37 kernel_fpu_begin(void)38void kernel_fpu_begin(void) 39 { 40 unsigned int *euen_curr; 41 42 if (!irqs_disabled()) 43 fpregs_lock(); 44 45 WARN_ON(this_cpu_read(in_kernel_fpu)); 46 47 this_cpu_write(in_kernel_fpu, true); 48 euen_curr = this_cpu_ptr(&euen_current); 49 50 *euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN); 51 52 #ifdef CONFIG_CPU_HAS_LASX 53 if (*euen_curr & CSR_EUEN_LASXEN) 54 _save_lasx(¤t->thread.fpu); 55 else 56 #endif 57 #ifdef CONFIG_CPU_HAS_LSX 58 if (*euen_curr & CSR_EUEN_LSXEN) 59 _save_lsx(¤t->thread.fpu); 60 else 61 #endif 62 if (*euen_curr & CSR_EUEN_FPEN) 63 _save_fp(¤t->thread.fpu); 64 65 write_fcsr(LOONGARCH_FCSR0, 0); 66 } 67 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 68 kernel_fpu_end(void)69void kernel_fpu_end(void) 70 { 71 unsigned int *euen_curr; 72 73 WARN_ON(!this_cpu_read(in_kernel_fpu)); 74 75 euen_curr = this_cpu_ptr(&euen_current); 76 77 #ifdef CONFIG_CPU_HAS_LASX 78 if (*euen_curr & CSR_EUEN_LASXEN) 79 _restore_lasx(¤t->thread.fpu); 80 else 81 #endif 82 #ifdef CONFIG_CPU_HAS_LSX 83 if (*euen_curr & CSR_EUEN_LSXEN) 84 _restore_lsx(¤t->thread.fpu); 85 else 86 #endif 87 if (*euen_curr & CSR_EUEN_FPEN) 88 _restore_fp(¤t->thread.fpu); 89 90 *euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN); 91 92 this_cpu_write(in_kernel_fpu, false); 93 94 if (!irqs_disabled()) 95 fpregs_unlock(); 96 } 97 EXPORT_SYMBOL_GPL(kernel_fpu_end); 98 init_euen_mask(void)99static int __init init_euen_mask(void) 100 { 101 if (cpu_has_lsx) 102 euen_mask |= CSR_EUEN_LSXEN; 103 104 if (cpu_has_lasx) 105 euen_mask |= CSR_EUEN_LASXEN; 106 107 return 0; 108 } 109 arch_initcall(init_euen_mask); 110