1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Author: Huacai Chen <chenhuacai@loongson.cn> 4 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 5 */ 6 #ifndef _ASM_FPU_H 7 #define _ASM_FPU_H 8 9 #include <linux/sched.h> 10 #include <linux/sched/task_stack.h> 11 #include <linux/ptrace.h> 12 #include <linux/thread_info.h> 13 #include <linux/bitops.h> 14 15 #include <asm/cpu.h> 16 #include <asm/cpu-features.h> 17 #include <asm/current.h> 18 #include <asm/loongarch.h> 19 #include <asm/processor.h> 20 #include <asm/ptrace.h> 21 22 struct sigcontext; 23 24 #define kernel_fpu_available() cpu_has_fpu 25 extern void kernel_fpu_begin(void); 26 extern void kernel_fpu_end(void); 27 28 extern void _init_fpu(unsigned int); 29 extern void _save_fp(struct loongarch_fpu *); 30 extern void _restore_fp(struct loongarch_fpu *); 31 32 extern void _save_lsx(struct loongarch_fpu *fpu); 33 extern void _restore_lsx(struct loongarch_fpu *fpu); 34 extern void _init_lsx_upper(void); 35 extern void _restore_lsx_upper(struct loongarch_fpu *fpu); 36 37 extern void _save_lasx(struct loongarch_fpu *fpu); 38 extern void _restore_lasx(struct loongarch_fpu *fpu); 39 extern void _init_lasx_upper(void); 40 extern void _restore_lasx_upper(struct loongarch_fpu *fpu); 41 42 static inline void enable_lsx(void); 43 static inline void disable_lsx(void); 44 static inline void save_lsx(struct task_struct *t); 45 static inline void restore_lsx(struct task_struct *t); 46 47 static inline void enable_lasx(void); 48 static inline void disable_lasx(void); 49 static inline void save_lasx(struct task_struct *t); 50 static inline void restore_lasx(struct task_struct *t); 51 52 /* 53 * Mask the FCSR Cause bits according to the Enable bits, observing 54 * that Unimplemented is always enabled. 55 */ 56 static inline unsigned long mask_fcsr_x(unsigned long fcsr) 57 { 58 return fcsr & ((fcsr & FPU_CSR_ALL_E) << 59 (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E))); 60 } 61 62 static inline int is_fp_enabled(void) 63 { 64 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_FPEN) ? 65 1 : 0; 66 } 67 68 static inline int is_lsx_enabled(void) 69 { 70 if (!cpu_has_lsx) 71 return 0; 72 73 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ? 74 1 : 0; 75 } 76 77 static inline int is_lasx_enabled(void) 78 { 79 if (!cpu_has_lasx) 80 return 0; 81 82 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ? 83 1 : 0; 84 } 85 86 static inline int is_simd_enabled(void) 87 { 88 return is_lsx_enabled() | is_lasx_enabled(); 89 } 90 91 #define enable_fpu() set_csr_euen(CSR_EUEN_FPEN) 92 93 #define disable_fpu() clear_csr_euen(CSR_EUEN_FPEN) 94 95 #define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU) 96 97 static inline int is_fpu_owner(void) 98 { 99 return test_thread_flag(TIF_USEDFPU); 100 } 101 102 static inline void __own_fpu(void) 103 { 104 enable_fpu(); 105 set_thread_flag(TIF_USEDFPU); 106 KSTK_EUEN(current) |= CSR_EUEN_FPEN; 107 } 108 109 static inline void own_fpu_inatomic(int restore) 110 { 111 if (cpu_has_fpu && !is_fpu_owner()) { 112 __own_fpu(); 113 if (restore) 114 _restore_fp(¤t->thread.fpu); 115 } 116 } 117 118 static inline void own_fpu(int restore) 119 { 120 preempt_disable(); 121 own_fpu_inatomic(restore); 122 preempt_enable(); 123 } 124 125 static inline void lose_fpu_inatomic(int save, struct task_struct *tsk) 126 { 127 if (is_fpu_owner()) { 128 if (!is_simd_enabled()) { 129 if (save) 130 _save_fp(&tsk->thread.fpu); 131 disable_fpu(); 132 } else { 133 if (save) { 134 if (!is_lasx_enabled()) 135 save_lsx(tsk); 136 else 137 save_lasx(tsk); 138 } 139 disable_fpu(); 140 disable_lsx(); 141 disable_lasx(); 142 clear_tsk_thread_flag(tsk, TIF_USEDSIMD); 143 } 144 clear_tsk_thread_flag(tsk, TIF_USEDFPU); 145 } 146 KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 147 } 148 149 static inline void lose_fpu(int save) 150 { 151 preempt_disable(); 152 lose_fpu_inatomic(save, current); 153 preempt_enable(); 154 } 155 156 static inline void init_fpu(void) 157 { 158 unsigned int fcsr = current->thread.fpu.fcsr; 159 160 __own_fpu(); 161 _init_fpu(fcsr); 162 set_used_math(); 163 } 164 165 static inline void save_fp(struct task_struct *tsk) 166 { 167 if (cpu_has_fpu) 168 _save_fp(&tsk->thread.fpu); 169 } 170 171 static inline void restore_fp(struct task_struct *tsk) 172 { 173 if (cpu_has_fpu) 174 _restore_fp(&tsk->thread.fpu); 175 } 176 177 static inline void save_fpu_regs(struct task_struct *tsk) 178 { 179 unsigned int euen; 180 181 if (tsk == current) { 182 preempt_disable(); 183 184 euen = csr_read32(LOONGARCH_CSR_EUEN); 185 186 #ifdef CONFIG_CPU_HAS_LASX 187 if (euen & CSR_EUEN_LASXEN) 188 _save_lasx(¤t->thread.fpu); 189 else 190 #endif 191 #ifdef CONFIG_CPU_HAS_LSX 192 if (euen & CSR_EUEN_LSXEN) 193 _save_lsx(¤t->thread.fpu); 194 else 195 #endif 196 if (euen & CSR_EUEN_FPEN) 197 _save_fp(¤t->thread.fpu); 198 199 preempt_enable(); 200 } 201 } 202 203 static inline int is_simd_owner(void) 204 { 205 return test_thread_flag(TIF_USEDSIMD); 206 } 207 208 #ifdef CONFIG_CPU_HAS_LSX 209 210 static inline void enable_lsx(void) 211 { 212 if (cpu_has_lsx) 213 csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); 214 } 215 216 static inline void disable_lsx(void) 217 { 218 if (cpu_has_lsx) 219 csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); 220 } 221 222 static inline void save_lsx(struct task_struct *t) 223 { 224 if (cpu_has_lsx) 225 _save_lsx(&t->thread.fpu); 226 } 227 228 static inline void restore_lsx(struct task_struct *t) 229 { 230 if (cpu_has_lsx) 231 _restore_lsx(&t->thread.fpu); 232 } 233 234 static inline void init_lsx_upper(void) 235 { 236 if (cpu_has_lsx) 237 _init_lsx_upper(); 238 } 239 240 static inline void restore_lsx_upper(struct task_struct *t) 241 { 242 if (cpu_has_lsx) 243 _restore_lsx_upper(&t->thread.fpu); 244 } 245 246 #else 247 static inline void enable_lsx(void) {} 248 static inline void disable_lsx(void) {} 249 static inline void save_lsx(struct task_struct *t) {} 250 static inline void restore_lsx(struct task_struct *t) {} 251 static inline void init_lsx_upper(void) {} 252 static inline void restore_lsx_upper(struct task_struct *t) {} 253 #endif 254 255 #ifdef CONFIG_CPU_HAS_LASX 256 257 static inline void enable_lasx(void) 258 { 259 260 if (cpu_has_lasx) 261 csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); 262 } 263 264 static inline void disable_lasx(void) 265 { 266 if (cpu_has_lasx) 267 csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); 268 } 269 270 static inline void save_lasx(struct task_struct *t) 271 { 272 if (cpu_has_lasx) 273 _save_lasx(&t->thread.fpu); 274 } 275 276 static inline void restore_lasx(struct task_struct *t) 277 { 278 if (cpu_has_lasx) 279 _restore_lasx(&t->thread.fpu); 280 } 281 282 static inline void init_lasx_upper(void) 283 { 284 if (cpu_has_lasx) 285 _init_lasx_upper(); 286 } 287 288 static inline void restore_lasx_upper(struct task_struct *t) 289 { 290 if (cpu_has_lasx) 291 _restore_lasx_upper(&t->thread.fpu); 292 } 293 294 #else 295 static inline void enable_lasx(void) {} 296 static inline void disable_lasx(void) {} 297 static inline void save_lasx(struct task_struct *t) {} 298 static inline void restore_lasx(struct task_struct *t) {} 299 static inline void init_lasx_upper(void) {} 300 static inline void restore_lasx_upper(struct task_struct *t) {} 301 #endif 302 303 static inline int thread_lsx_context_live(void) 304 { 305 if (!cpu_has_lsx) 306 return 0; 307 308 return test_thread_flag(TIF_LSX_CTX_LIVE); 309 } 310 311 static inline int thread_lasx_context_live(void) 312 { 313 if (!cpu_has_lasx) 314 return 0; 315 316 return test_thread_flag(TIF_LASX_CTX_LIVE); 317 } 318 319 #endif /* _ASM_FPU_H */ 320