1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 *
33 * $NetBSD: fpu.c,v 1.5 2001/07/22 11:29:46 wiz Exp $
34 */
35
36 #include <sys/param.h>
37 #include <sys/proc.h>
38 #include <sys/systm.h>
39 #include <sys/limits.h>
40
41 #include <machine/fpu.h>
42 #include <machine/pcb.h>
43 #include <machine/psl.h>
44 #include <machine/altivec.h>
45
46 static void
save_fpu_int(struct thread * td)47 save_fpu_int(struct thread *td)
48 {
49 register_t msr;
50 struct pcb *pcb;
51
52 pcb = td->td_pcb;
53
54 /*
55 * Temporarily re-enable floating-point during the save
56 */
57 msr = mfmsr();
58 if (pcb->pcb_flags & PCB_VSX)
59 mtmsr(msr | PSL_FP | PSL_VSX);
60 else
61 mtmsr(msr | PSL_FP);
62
63 /*
64 * Save the floating-point registers and FPSCR to the PCB
65 */
66 if (pcb->pcb_flags & PCB_VSX) {
67 #if _BYTE_ORDER == _BIG_ENDIAN
68 #define SFP(n) __asm("stxvw4x " #n ", 0,%0" \
69 :: "b"(&pcb->pcb_fpu.fpr[n]));
70 #else
71 /*
72 * stxvw2x will swap words within the FP double word on LE systems,
73 * leading to corruption if VSX is used to store state and FP is
74 * subsequently used to restore state.
75 * Use stxvd2x instead.
76 */
77 #define SFP(n) __asm("stxvd2x " #n ", 0,%0" \
78 :: "b"(&pcb->pcb_fpu.fpr[n]));
79 #endif
80 SFP(0); SFP(1); SFP(2); SFP(3);
81 SFP(4); SFP(5); SFP(6); SFP(7);
82 SFP(8); SFP(9); SFP(10); SFP(11);
83 SFP(12); SFP(13); SFP(14); SFP(15);
84 SFP(16); SFP(17); SFP(18); SFP(19);
85 SFP(20); SFP(21); SFP(22); SFP(23);
86 SFP(24); SFP(25); SFP(26); SFP(27);
87 SFP(28); SFP(29); SFP(30); SFP(31);
88 #undef SFP
89 } else {
90 #define SFP(n) __asm("stfd " #n ", 0(%0)" \
91 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
92 SFP(0); SFP(1); SFP(2); SFP(3);
93 SFP(4); SFP(5); SFP(6); SFP(7);
94 SFP(8); SFP(9); SFP(10); SFP(11);
95 SFP(12); SFP(13); SFP(14); SFP(15);
96 SFP(16); SFP(17); SFP(18); SFP(19);
97 SFP(20); SFP(21); SFP(22); SFP(23);
98 SFP(24); SFP(25); SFP(26); SFP(27);
99 SFP(28); SFP(29); SFP(30); SFP(31);
100 #undef SFP
101 }
102 __asm __volatile ("mffs 0; stfd 0,0(%0)" :: "b"(&pcb->pcb_fpu.fpscr));
103
104 /*
105 * Disable floating-point again
106 */
107 isync();
108 mtmsr(msr);
109 }
110
111 void
enable_fpu(struct thread * td)112 enable_fpu(struct thread *td)
113 {
114 register_t msr;
115 struct pcb *pcb;
116 struct trapframe *tf;
117
118 pcb = td->td_pcb;
119 tf = trapframe(td);
120
121 /*
122 * Save the thread's FPU CPU number, and set the CPU's current
123 * FPU thread
124 */
125 td->td_pcb->pcb_fpcpu = PCPU_GET(cpuid);
126 PCPU_SET(fputhread, td);
127
128 /*
129 * Enable the FPU for when the thread returns from the exception.
130 * If this is the first time the FPU has been used by the thread,
131 * initialise the FPU registers and FPSCR to 0, and set the flag
132 * to indicate that the FPU is in use.
133 */
134 pcb->pcb_flags |= PCB_FPU;
135 if (pcb->pcb_flags & PCB_VSX)
136 tf->srr1 |= PSL_FP | PSL_VSX;
137 else
138 tf->srr1 |= PSL_FP;
139 if (!(pcb->pcb_flags & PCB_FPREGS)) {
140 memset(&pcb->pcb_fpu, 0, sizeof pcb->pcb_fpu);
141 pcb->pcb_flags |= PCB_FPREGS;
142 }
143
144 /*
145 * Temporarily enable floating-point so the registers
146 * can be restored.
147 */
148 msr = mfmsr();
149 if (pcb->pcb_flags & PCB_VSX)
150 mtmsr(msr | PSL_FP | PSL_VSX);
151 else
152 mtmsr(msr | PSL_FP);
153
154 /*
155 * Load the floating point registers and FPSCR from the PCB.
156 * (A value of 0xff for mtfsf specifies that all 8 4-bit fields
157 * of the saved FPSCR are to be loaded from the FPU reg).
158 */
159 __asm __volatile ("lfd 0,0(%0); mtfsf 0xff,0"
160 :: "b"(&pcb->pcb_fpu.fpscr));
161
162 if (pcb->pcb_flags & PCB_VSX) {
163 #if _BYTE_ORDER == _BIG_ENDIAN
164 #define LFP(n) __asm("lxvw4x " #n ", 0,%0" \
165 :: "b"(&pcb->pcb_fpu.fpr[n]));
166 #else
167 /*
168 * lxvw4x will swap words within the FP double word on LE systems,
169 * leading to corruption if FP is used to store state and VSX is
170 * subsequently used to restore state.
171 * Use lxvd2x instead.
172 */
173 #define LFP(n) __asm("lxvd2x " #n ", 0,%0" \
174 :: "b"(&pcb->pcb_fpu.fpr[n]));
175 #endif
176 LFP(0); LFP(1); LFP(2); LFP(3);
177 LFP(4); LFP(5); LFP(6); LFP(7);
178 LFP(8); LFP(9); LFP(10); LFP(11);
179 LFP(12); LFP(13); LFP(14); LFP(15);
180 LFP(16); LFP(17); LFP(18); LFP(19);
181 LFP(20); LFP(21); LFP(22); LFP(23);
182 LFP(24); LFP(25); LFP(26); LFP(27);
183 LFP(28); LFP(29); LFP(30); LFP(31);
184 #undef LFP
185 } else {
186 #define LFP(n) __asm("lfd " #n ", 0(%0)" \
187 :: "b"(&pcb->pcb_fpu.fpr[n].fpr));
188 LFP(0); LFP(1); LFP(2); LFP(3);
189 LFP(4); LFP(5); LFP(6); LFP(7);
190 LFP(8); LFP(9); LFP(10); LFP(11);
191 LFP(12); LFP(13); LFP(14); LFP(15);
192 LFP(16); LFP(17); LFP(18); LFP(19);
193 LFP(20); LFP(21); LFP(22); LFP(23);
194 LFP(24); LFP(25); LFP(26); LFP(27);
195 LFP(28); LFP(29); LFP(30); LFP(31);
196 #undef LFP
197 }
198
199 isync();
200 mtmsr(msr);
201 }
202
203 void
save_fpu(struct thread * td)204 save_fpu(struct thread *td)
205 {
206 struct pcb *pcb;
207
208 pcb = td->td_pcb;
209
210 save_fpu_int(td);
211
212 /*
213 * Clear the current fp thread and pcb's CPU id
214 * XXX should this be left clear to allow lazy save/restore ?
215 */
216 pcb->pcb_fpcpu = INT_MAX;
217 PCPU_SET(fputhread, NULL);
218 }
219
220 /*
221 * Save fpu state without dropping ownership. This will only save state if
222 * the current fpu thread is `td'.
223 */
224 void
save_fpu_nodrop(struct thread * td)225 save_fpu_nodrop(struct thread *td)
226 {
227
228 if (td == PCPU_GET(fputhread))
229 save_fpu_int(td);
230 }
231
232 /*
233 * Clear Floating-Point Status and Control Register
234 */
235 void
cleanup_fpscr(void)236 cleanup_fpscr(void)
237 {
238 register_t msr;
239
240 msr = mfmsr();
241 mtmsr(msr | PSL_FP);
242 mtfsf(0);
243
244 isync();
245 mtmsr(msr);
246 }
247
248 /*
249 * Get the current fp exception
250 */
251 u_int
get_fpu_exception(struct thread * td)252 get_fpu_exception(struct thread *td)
253 {
254 register_t msr;
255 u_int ucode;
256 register_t reg;
257
258 critical_enter();
259
260 msr = mfmsr();
261 mtmsr(msr | PSL_FP);
262
263 reg = mffs();
264
265 isync();
266 mtmsr(msr);
267
268 critical_exit();
269
270 if (reg & FPSCR_ZX)
271 ucode = FPE_FLTDIV;
272 else if (reg & FPSCR_OX)
273 ucode = FPE_FLTOVF;
274 else if (reg & FPSCR_UX)
275 ucode = FPE_FLTUND;
276 else if (reg & FPSCR_XX)
277 ucode = FPE_FLTRES;
278 else
279 ucode = FPE_FLTINV;
280
281 return ucode;
282 }
283
284 void
enable_fpu_kern(void)285 enable_fpu_kern(void)
286 {
287 register_t msr;
288
289 msr = mfmsr() | PSL_FP;
290
291 if (cpu_features & PPC_FEATURE_HAS_VSX)
292 msr |= PSL_VSX;
293
294 mtmsr(msr);
295 }
296
297 void
disable_fpu(struct thread * td)298 disable_fpu(struct thread *td)
299 {
300 register_t msr;
301 struct pcb *pcb;
302 struct trapframe *tf;
303
304 pcb = td->td_pcb;
305 tf = trapframe(td);
306
307 /* Disable FPU in kernel (if enabled) */
308 msr = mfmsr() & ~(PSL_FP | PSL_VSX);
309 isync();
310 mtmsr(msr);
311
312 /*
313 * Disable FPU in userspace. It will be re-enabled when
314 * an FP or VSX instruction is executed.
315 */
316 tf->srr1 &= ~(PSL_FP | PSL_VSX);
317 pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX);
318 }
319
320 #ifndef __SPE__
321 /*
322 * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and
323 * fpu_kern_leave can handle !FPU_KERN_NOCTX.
324 */
325 struct fpu_kern_ctx {
326 #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */
327 #define FPU_KERN_CTX_INUSE 0x02
328 uint32_t flags;
329 };
330
331 void
fpu_kern_enter(struct thread * td,struct fpu_kern_ctx * ctx,u_int flags)332 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
333 {
334 struct pcb *pcb;
335
336 pcb = td->td_pcb;
337
338 KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL,
339 ("ctx is required when !FPU_KERN_NOCTX"));
340 KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0,
341 ("using inuse ctx"));
342 KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0,
343 ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state"));
344
345 if ((flags & FPU_KERN_NOCTX) != 0) {
346 critical_enter();
347
348 if (pcb->pcb_flags & PCB_FPU) {
349 save_fpu(td);
350 pcb->pcb_flags |= PCB_FPREGS;
351 }
352 enable_fpu_kern();
353
354 if (pcb->pcb_flags & PCB_VEC) {
355 save_vec(td);
356 pcb->pcb_flags |= PCB_VECREGS;
357 }
358 enable_vec_kern();
359
360 pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE;
361 return;
362 }
363
364 KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!"));
365 }
366
367 int
fpu_kern_leave(struct thread * td,struct fpu_kern_ctx * ctx)368 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
369 {
370 struct pcb *pcb;
371
372 pcb = td->td_pcb;
373
374 if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) {
375 KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX"));
376 KASSERT(PCPU_GET(fpcurthread) == NULL,
377 ("non-NULL fpcurthread for PCB_FP_NOSAVE"));
378 CRITICAL_ASSERT(td);
379
380 /* Disable FPU, VMX, and VSX */
381 disable_fpu(td);
382 disable_vec(td);
383
384 pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE;
385
386 critical_exit();
387 } else {
388 KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!"));
389 }
390
391 pcb->pcb_flags &= ~PCB_KERN_FPU;
392
393 return 0;
394 }
395
396 int
is_fpu_kern_thread(u_int flags __unused)397 is_fpu_kern_thread(u_int flags __unused)
398 {
399 struct pcb *curpcb;
400
401 if ((curthread->td_pflags & TDP_KTHREAD) == 0)
402 return (0);
403 curpcb = curthread->td_pcb;
404 return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0);
405 }
406
407 #endif /* !__SPE__ */
408