1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/init.h> 8 #include <asm/fpu.h> 9 #include <asm/smp.h> 10 11 static unsigned int euen_mask = CSR_EUEN_FPEN; 12 13 /* 14 * The critical section between kernel_fpu_begin() and kernel_fpu_end() 15 * is non-reentrant. It is the caller's responsibility to avoid reentrance. 16 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example. 17 */ 18 static DEFINE_PER_CPU(bool, in_kernel_fpu); 19 static DEFINE_PER_CPU(unsigned int, euen_current); 20 21 void kernel_fpu_begin(void) 22 { 23 unsigned int *euen_curr; 24 25 preempt_disable(); 26 27 WARN_ON(this_cpu_read(in_kernel_fpu)); 28 29 this_cpu_write(in_kernel_fpu, true); 30 euen_curr = this_cpu_ptr(&euen_current); 31 32 *euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN); 33 34 #ifdef CONFIG_CPU_HAS_LASX 35 if (*euen_curr & CSR_EUEN_LASXEN) 36 _save_lasx(¤t->thread.fpu); 37 else 38 #endif 39 #ifdef CONFIG_CPU_HAS_LSX 40 if (*euen_curr & CSR_EUEN_LSXEN) 41 _save_lsx(¤t->thread.fpu); 42 else 43 #endif 44 if (*euen_curr & CSR_EUEN_FPEN) 45 _save_fp(¤t->thread.fpu); 46 47 write_fcsr(LOONGARCH_FCSR0, 0); 48 } 49 EXPORT_SYMBOL_GPL(kernel_fpu_begin); 50 51 void kernel_fpu_end(void) 52 { 53 unsigned int *euen_curr; 54 55 WARN_ON(!this_cpu_read(in_kernel_fpu)); 56 57 euen_curr = this_cpu_ptr(&euen_current); 58 59 #ifdef CONFIG_CPU_HAS_LASX 60 if (*euen_curr & CSR_EUEN_LASXEN) 61 _restore_lasx(¤t->thread.fpu); 62 else 63 #endif 64 #ifdef CONFIG_CPU_HAS_LSX 65 if (*euen_curr & CSR_EUEN_LSXEN) 66 _restore_lsx(¤t->thread.fpu); 67 else 68 #endif 69 if (*euen_curr & CSR_EUEN_FPEN) 70 _restore_fp(¤t->thread.fpu); 71 72 *euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN); 73 74 this_cpu_write(in_kernel_fpu, false); 75 76 preempt_enable(); 77 } 78 EXPORT_SYMBOL_GPL(kernel_fpu_end); 79 80 static int __init init_euen_mask(void) 81 { 82 if (cpu_has_lsx) 83 euen_mask |= CSR_EUEN_LSXEN; 84 85 if (cpu_has_lasx) 86 euen_mask |= CSR_EUEN_LASXEN; 87 88 return 0; 89 } 90 arch_initcall(init_euen_mask); 91