1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 *
33 * $NetBSD: fpu.c,v 1.5 2001/07/22 11:29:46 wiz Exp $
34 */
35
36 #include <sys/param.h>
37 #include <sys/proc.h>
38 #include <sys/systm.h>
39 #include <sys/limits.h>
40
41 #include <machine/fpu.h>
42 #include <machine/pcb.h>
43 #include <machine/psl.h>
44 #include <machine/altivec.h>
45
46 static void
save_fpu_int(struct thread * td)47 save_fpu_int(struct thread *td)
48 {
49 register_t msr;
50 struct pcb *pcb;
51
52 pcb = td->td_pcb;
53
54 /*
55 * Temporarily re-enable floating-point during the save
56 */
57 msr = mfmsr();
58 if (pcb->pcb_flags & PCB_VSX)
59 mtmsr(msr | PSL_FP | PSL_VSX);
60 else
61 mtmsr(msr | PSL_FP);
62
63 /*
64 * Save the floating-point registers and FPSCR to the PCB
65 */
66 if (pcb->pcb_flags & PCB_VSX) {
67 #define SFP(n) __asm ("stxvw4x " #n ", 0,%0" \
68 :: "b"(&pcb->pcb_fpu.fpr[n]));
69 SFP(0); SFP(1); SFP(2); SFP(3);
70 SFP(4); SFP(5); SFP(6); SFP(7);
71 SFP(8); SFP(9); SFP(10); SFP(11);
72 SFP(12); SFP(13); SFP(14); SFP(15);
73 SFP(16); SFP(17); SFP(18); SFP(19);
74 SFP(20); SFP(21); SFP(22); SFP(23);
75 SFP(24); SFP(25); SFP(26); SFP(27);
76 SFP(28); SFP(29); SFP(30); SFP(31);
77 #undef SFP
78 } else {
79 #define SFP(n) __asm ("stfd " #n ", 0(%0)" \
80 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
81 SFP(0); SFP(1); SFP(2); SFP(3);
82 SFP(4); SFP(5); SFP(6); SFP(7);
83 SFP(8); SFP(9); SFP(10); SFP(11);
84 SFP(12); SFP(13); SFP(14); SFP(15);
85 SFP(16); SFP(17); SFP(18); SFP(19);
86 SFP(20); SFP(21); SFP(22); SFP(23);
87 SFP(24); SFP(25); SFP(26); SFP(27);
88 SFP(28); SFP(29); SFP(30); SFP(31);
89 #undef SFP
90 }
91 __asm __volatile ("mffs 0; stfd 0,0(%0)" :: "b"(&pcb->pcb_fpu.fpscr));
92
93 /*
94 * Disable floating-point again
95 */
96 isync();
97 mtmsr(msr);
98 }
99
100 void
enable_fpu(struct thread * td)101 enable_fpu(struct thread *td)
102 {
103 register_t msr;
104 struct pcb *pcb;
105 struct trapframe *tf;
106
107 pcb = td->td_pcb;
108 tf = trapframe(td);
109
110 /*
111 * Save the thread's FPU CPU number, and set the CPU's current
112 * FPU thread
113 */
114 td->td_pcb->pcb_fpcpu = PCPU_GET(cpuid);
115 PCPU_SET(fputhread, td);
116
117 /*
118 * Enable the FPU for when the thread returns from the exception.
119 * If this is the first time the FPU has been used by the thread,
120 * initialise the FPU registers and FPSCR to 0, and set the flag
121 * to indicate that the FPU is in use.
122 */
123 pcb->pcb_flags |= PCB_FPU;
124 if (pcb->pcb_flags & PCB_VSX)
125 tf->srr1 |= PSL_FP | PSL_VSX;
126 else
127 tf->srr1 |= PSL_FP;
128 if (!(pcb->pcb_flags & PCB_FPREGS)) {
129 memset(&pcb->pcb_fpu, 0, sizeof pcb->pcb_fpu);
130 pcb->pcb_flags |= PCB_FPREGS;
131 }
132
133 /*
134 * Temporarily enable floating-point so the registers
135 * can be restored.
136 */
137 msr = mfmsr();
138 if (pcb->pcb_flags & PCB_VSX)
139 mtmsr(msr | PSL_FP | PSL_VSX);
140 else
141 mtmsr(msr | PSL_FP);
142
143 /*
144 * Load the floating point registers and FPSCR from the PCB.
145 * (A value of 0xff for mtfsf specifies that all 8 4-bit fields
146 * of the saved FPSCR are to be loaded from the FPU reg).
147 */
148 __asm __volatile ("lfd 0,0(%0); mtfsf 0xff,0"
149 :: "b"(&pcb->pcb_fpu.fpscr));
150
151 if (pcb->pcb_flags & PCB_VSX) {
152 #define LFP(n) __asm ("lxvw4x " #n ", 0,%0" \
153 :: "b"(&pcb->pcb_fpu.fpr[n]));
154 LFP(0); LFP(1); LFP(2); LFP(3);
155 LFP(4); LFP(5); LFP(6); LFP(7);
156 LFP(8); LFP(9); LFP(10); LFP(11);
157 LFP(12); LFP(13); LFP(14); LFP(15);
158 LFP(16); LFP(17); LFP(18); LFP(19);
159 LFP(20); LFP(21); LFP(22); LFP(23);
160 LFP(24); LFP(25); LFP(26); LFP(27);
161 LFP(28); LFP(29); LFP(30); LFP(31);
162 #undef LFP
163 } else {
164 #define LFP(n) __asm ("lfd " #n ", 0(%0)" \
165 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
166 LFP(0); LFP(1); LFP(2); LFP(3);
167 LFP(4); LFP(5); LFP(6); LFP(7);
168 LFP(8); LFP(9); LFP(10); LFP(11);
169 LFP(12); LFP(13); LFP(14); LFP(15);
170 LFP(16); LFP(17); LFP(18); LFP(19);
171 LFP(20); LFP(21); LFP(22); LFP(23);
172 LFP(24); LFP(25); LFP(26); LFP(27);
173 LFP(28); LFP(29); LFP(30); LFP(31);
174 #undef LFP
175 }
176
177 isync();
178 mtmsr(msr);
179 }
180
181 void
save_fpu(struct thread * td)182 save_fpu(struct thread *td)
183 {
184 struct pcb *pcb;
185
186 pcb = td->td_pcb;
187
188 save_fpu_int(td);
189
190 /*
191 * Clear the current fp thread and pcb's CPU id
192 * XXX should this be left clear to allow lazy save/restore ?
193 */
194 pcb->pcb_fpcpu = INT_MAX;
195 PCPU_SET(fputhread, NULL);
196 }
197
198 /*
199 * Save fpu state without dropping ownership. This will only save state if
200 * the current fpu thread is `td'.
201 */
202 void
save_fpu_nodrop(struct thread * td)203 save_fpu_nodrop(struct thread *td)
204 {
205
206 if (td == PCPU_GET(fputhread))
207 save_fpu_int(td);
208 }
209
210 /*
211 * Clear Floating-Point Status and Control Register
212 */
213 void
cleanup_fpscr(void)214 cleanup_fpscr(void)
215 {
216 register_t msr;
217
218 msr = mfmsr();
219 mtmsr(msr | PSL_FP);
220 mtfsf(0);
221
222 isync();
223 mtmsr(msr);
224 }
225
226 /*
227 * Get the current fp exception
228 */
229 u_int
get_fpu_exception(struct thread * td)230 get_fpu_exception(struct thread *td)
231 {
232 register_t msr;
233 u_int ucode;
234 register_t reg;
235
236 critical_enter();
237
238 msr = mfmsr();
239 mtmsr(msr | PSL_FP);
240
241 reg = mffs();
242
243 isync();
244 mtmsr(msr);
245
246 critical_exit();
247
248 if (reg & FPSCR_ZX)
249 ucode = FPE_FLTDIV;
250 else if (reg & FPSCR_OX)
251 ucode = FPE_FLTOVF;
252 else if (reg & FPSCR_UX)
253 ucode = FPE_FLTUND;
254 else if (reg & FPSCR_XX)
255 ucode = FPE_FLTRES;
256 else
257 ucode = FPE_FLTINV;
258
259 return ucode;
260 }
261
262 void
enable_fpu_kern(void)263 enable_fpu_kern(void)
264 {
265 register_t msr;
266
267 msr = mfmsr() | PSL_FP;
268
269 if (cpu_features & PPC_FEATURE_HAS_VSX)
270 msr |= PSL_VSX;
271
272 mtmsr(msr);
273 }
274
275 void
disable_fpu(struct thread * td)276 disable_fpu(struct thread *td)
277 {
278 register_t msr;
279 struct pcb *pcb;
280 struct trapframe *tf;
281
282 pcb = td->td_pcb;
283 tf = trapframe(td);
284
285 /* Disable FPU in kernel (if enabled) */
286 msr = mfmsr() & ~(PSL_FP | PSL_VSX);
287 isync();
288 mtmsr(msr);
289
290 /*
291 * Disable FPU in userspace. It will be re-enabled when
292 * an FP or VSX instruction is executed.
293 */
294 tf->srr1 &= ~(PSL_FP | PSL_VSX);
295 pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX);
296 }
297
298 #ifndef __SPE__
299 /*
300 * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and
301 * fpu_kern_leave can handle !FPU_KERN_NOCTX.
302 */
303 struct fpu_kern_ctx {
304 #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */
305 #define FPU_KERN_CTX_INUSE 0x02
306 uint32_t flags;
307 };
308
309 void
fpu_kern_enter(struct thread * td,struct fpu_kern_ctx * ctx,u_int flags)310 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
311 {
312 struct pcb *pcb;
313
314 pcb = td->td_pcb;
315
316 KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL,
317 ("ctx is required when !FPU_KERN_NOCTX"));
318 KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
319 ("using inuse ctx"));
320 KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0,
321 ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state"));
322
323 if ((flags & FPU_KERN_NOCTX) != 0) {
324 critical_enter();
325
326 if (pcb->pcb_flags & PCB_FPU) {
327 save_fpu(td);
328 pcb->pcb_flags |= PCB_FPREGS;
329 }
330 enable_fpu_kern();
331
332 if (pcb->pcb_flags & PCB_VEC) {
333 save_vec(td);
334 pcb->pcb_flags |= PCB_VECREGS;
335 }
336 enable_vec_kern();
337
338 pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE;
339 return;
340 }
341
342 KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!"));
343 }
344
345 int
fpu_kern_leave(struct thread * td,struct fpu_kern_ctx * ctx)346 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
347 {
348 struct pcb *pcb;
349
350 pcb = td->td_pcb;
351
352 if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) {
353 KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX"));
354 KASSERT(PCPU_GET(fpcurthread) == NULL,
355 ("non-NULL fpcurthread for PCB_FP_NOSAVE"));
356 CRITICAL_ASSERT(td);
357
358 /* Disable FPU, VMX, and VSX */
359 disable_fpu(td);
360 disable_vec(td);
361
362 pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE;
363
364 critical_exit();
365 } else {
366 KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!"));
367 }
368
369 pcb->pcb_flags &= ~PCB_KERN_FPU;
370
371 return 0;
372 }
373
374 int
is_fpu_kern_thread(u_int flags __unused)375 is_fpu_kern_thread(u_int flags __unused)
376 {
377 struct pcb *curpcb;
378
379 if ((curthread->td_pflags & TDP_KTHREAD) == 0)
380 return (0);
381 curpcb = curthread->td_pcb;
382 return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0);
383 }
384
385 #endif /* !__SPE__ */
386