15b81b6b3SRodney W. Grimes /*- 25b81b6b3SRodney W. Grimes * Copyright (c) 1990 William Jolitz. 35b81b6b3SRodney W. Grimes * Copyright (c) 1991 The Regents of the University of California. 45b81b6b3SRodney W. Grimes * All rights reserved. 55b81b6b3SRodney W. Grimes * 65b81b6b3SRodney W. Grimes * Redistribution and use in source and binary forms, with or without 75b81b6b3SRodney W. Grimes * modification, are permitted provided that the following conditions 85b81b6b3SRodney W. Grimes * are met: 95b81b6b3SRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 105b81b6b3SRodney W. Grimes * notice, this list of conditions and the following disclaimer. 115b81b6b3SRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 125b81b6b3SRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 135b81b6b3SRodney W. Grimes * documentation and/or other materials provided with the distribution. 145b81b6b3SRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 155b81b6b3SRodney W. Grimes * may be used to endorse or promote products derived from this software 165b81b6b3SRodney W. Grimes * without specific prior written permission. 175b81b6b3SRodney W. Grimes * 185b81b6b3SRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 195b81b6b3SRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 205b81b6b3SRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 215b81b6b3SRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 225b81b6b3SRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 235b81b6b3SRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 245b81b6b3SRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 255b81b6b3SRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 265b81b6b3SRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 275b81b6b3SRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 285b81b6b3SRodney W. Grimes * SUCH DAMAGE. 295b81b6b3SRodney W. Grimes * 3021616ec3SPeter Wemm * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 315b81b6b3SRodney W. Grimes */ 325b81b6b3SRodney W. Grimes 3356ae44c5SDavid E. O'Brien #include <sys/cdefs.h> 3456ae44c5SDavid E. O'Brien __FBSDID("$FreeBSD$"); 3556ae44c5SDavid E. O'Brien 36f540b106SGarrett Wollman #include <sys/param.h> 37f540b106SGarrett Wollman #include <sys/systm.h> 386182fdbdSPeter Wemm #include <sys/bus.h> 393a34a5c3SPoul-Henning Kamp #include <sys/kernel.h> 40fb919e4dSMark Murray #include <sys/lock.h> 41cd59d49dSBruce Evans #include <sys/malloc.h> 426182fdbdSPeter Wemm #include <sys/module.h> 43c1ef8aacSJake Burkholder #include <sys/mutex.h> 44fb919e4dSMark Murray #include <sys/mutex.h> 45fb919e4dSMark Murray #include <sys/proc.h> 46fb919e4dSMark Murray #include <sys/sysctl.h> 476182fdbdSPeter Wemm #include <machine/bus.h> 486182fdbdSPeter Wemm #include <sys/rman.h> 49663f1485SBruce Evans #include <sys/signalvar.h> 502741efecSPeter Grehan #include <vm/uma.h> 512f86936aSGarrett Wollman 527f47cf2fSBruce Evans #include <machine/cputypes.h> 537f47cf2fSBruce Evans #include <machine/frame.h> 540d2a2989SPeter Wemm #include <machine/intr_machdep.h> 55c673fe98SBruce Evans #include <machine/md_var.h> 565400ed3bSPeter Wemm #include <machine/pcb.h> 577f47cf2fSBruce Evans #include <machine/psl.h> 586182fdbdSPeter Wemm #include <machine/resource.h> 59f540b106SGarrett Wollman #include <machine/specialreg.h> 607f47cf2fSBruce Evans #include <machine/segments.h> 6130abe507SJonathan Mini #include <machine/ucontext.h> 622f86936aSGarrett Wollman 635b81b6b3SRodney W. Grimes /* 64bf2f09eeSPeter Wemm * Floating point support. 655b81b6b3SRodney W. Grimes */ 665b81b6b3SRodney W. Grimes 67a5f50ef9SJoerg Wunsch #if defined(__GNUCLIKE_ASM) && !defined(lint) 685b81b6b3SRodney W. Grimes 6917275403SJung-uk Kim #define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) 7030402401SJung-uk Kim #define fnclex() __asm __volatile("fnclex") 7130402401SJung-uk Kim #define fninit() __asm __volatile("fninit") 721d37f051SBruce Evans #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) 732e50fa36SJung-uk Kim #define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) 7430402401SJung-uk Kim #define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) 759d146ac5SPeter Wemm #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) 7607c86dcfSJung-uk Kim #define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) 77a81f9fedSKonstantin Belousov #define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) 785b81b6b3SRodney W. Grimes 7994818d19SKonstantin Belousov static __inline void 8094818d19SKonstantin Belousov xrstor(char *addr, uint64_t mask) 8194818d19SKonstantin Belousov { 8294818d19SKonstantin Belousov uint32_t low, hi; 8394818d19SKonstantin Belousov 8494818d19SKonstantin Belousov low = mask; 8594818d19SKonstantin Belousov hi = mask >> 32; 867574a595SJohn Baldwin __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); 8794818d19SKonstantin Belousov } 8894818d19SKonstantin Belousov 8994818d19SKonstantin Belousov static __inline void 9094818d19SKonstantin Belousov xsave(char *addr, uint64_t mask) 9194818d19SKonstantin Belousov { 9294818d19SKonstantin Belousov uint32_t low, hi; 9394818d19SKonstantin Belousov 9494818d19SKonstantin Belousov low = mask; 9594818d19SKonstantin Belousov hi = mask >> 32; 967574a595SJohn Baldwin __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : 977574a595SJohn Baldwin "memory"); 9894818d19SKonstantin Belousov } 9994818d19SKonstantin Belousov 100cf4e1c46SPeter Wemm #else /* !(__GNUCLIKE_ASM && !lint) */ 1015b81b6b3SRodney W. Grimes 10217275403SJung-uk Kim void fldcw(u_short cw); 10389c9a483SAlfred Perlstein void fnclex(void); 10489c9a483SAlfred Perlstein void fninit(void); 10589c9a483SAlfred Perlstein void fnstcw(caddr_t addr); 10689c9a483SAlfred Perlstein void fnstsw(caddr_t addr); 10789c9a483SAlfred Perlstein void fxsave(caddr_t addr); 10889c9a483SAlfred Perlstein void fxrstor(caddr_t addr); 10907c86dcfSJung-uk Kim void ldmxcsr(u_int csr); 110a42fa0afSKonstantin Belousov void stmxcsr(u_int *csr); 11194818d19SKonstantin Belousov void xrstor(char *addr, uint64_t mask); 11294818d19SKonstantin Belousov void xsave(char *addr, uint64_t mask); 1135b81b6b3SRodney W. Grimes 114cf4e1c46SPeter Wemm #endif /* __GNUCLIKE_ASM && !lint */ 1155b81b6b3SRodney W. Grimes 116d706ec29SJohn Baldwin #define start_emulating() load_cr0(rcr0() | CR0_TS) 117d706ec29SJohn Baldwin #define stop_emulating() clts() 118d706ec29SJohn Baldwin 1198c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct savefpu) == 512); 1208c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct xstate_hdr) == 64); 1218c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct savefpu_ymm) == 832); 1228c6f8f3dSKonstantin Belousov 1238c6f8f3dSKonstantin Belousov /* 1248c6f8f3dSKonstantin Belousov * This requirement is to make it easier for asm code to calculate 1258c6f8f3dSKonstantin Belousov * offset of the fpu save area from the pcb address. FPU save area 126b74a2290SKonstantin Belousov * must be 64-byte aligned. 1278c6f8f3dSKonstantin Belousov */ 1288c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0); 1295b81b6b3SRodney W. Grimes 1302652af56SColin Percival static void fpu_clean_state(void); 1312652af56SColin Percival 1320b7dc0a7SJohn Baldwin SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, 1330b7dc0a7SJohn Baldwin NULL, 1, "Floating point instructions executed in hardware"); 1343a34a5c3SPoul-Henning Kamp 1358c6f8f3dSKonstantin Belousov int use_xsave; /* non-static for cpu_switch.S */ 1368c6f8f3dSKonstantin Belousov uint64_t xsave_mask; /* the same */ 1372741efecSPeter Grehan static uma_zone_t fpu_save_area_zone; 1388c6f8f3dSKonstantin Belousov static struct savefpu *fpu_initialstate; 1398c6f8f3dSKonstantin Belousov 140333d0c60SKonstantin Belousov struct xsave_area_elm_descr { 141333d0c60SKonstantin Belousov u_int offset; 142333d0c60SKonstantin Belousov u_int size; 143333d0c60SKonstantin Belousov } *xsave_area_desc; 144333d0c60SKonstantin Belousov 1458c6f8f3dSKonstantin Belousov void 1468c6f8f3dSKonstantin Belousov fpusave(void *addr) 1478c6f8f3dSKonstantin Belousov { 1488c6f8f3dSKonstantin Belousov 1498c6f8f3dSKonstantin Belousov if (use_xsave) 1508c6f8f3dSKonstantin Belousov xsave((char *)addr, xsave_mask); 1518c6f8f3dSKonstantin Belousov else 1528c6f8f3dSKonstantin Belousov fxsave((char *)addr); 1538c6f8f3dSKonstantin Belousov } 1548c6f8f3dSKonstantin Belousov 1552741efecSPeter Grehan void 1568c6f8f3dSKonstantin Belousov fpurestore(void *addr) 1578c6f8f3dSKonstantin Belousov { 1588c6f8f3dSKonstantin Belousov 1598c6f8f3dSKonstantin Belousov if (use_xsave) 1608c6f8f3dSKonstantin Belousov xrstor((char *)addr, xsave_mask); 1618c6f8f3dSKonstantin Belousov else 1628c6f8f3dSKonstantin Belousov fxrstor((char *)addr); 1638c6f8f3dSKonstantin Belousov } 1643902c3efSSteve Passe 1655b81b6b3SRodney W. Grimes /* 1668c6f8f3dSKonstantin Belousov * Enable XSAVE if supported and allowed by user. 1678c6f8f3dSKonstantin Belousov * Calculate the xsave_mask. 1688c6f8f3dSKonstantin Belousov */ 1698c6f8f3dSKonstantin Belousov static void 1708c6f8f3dSKonstantin Belousov fpuinit_bsp1(void) 1718c6f8f3dSKonstantin Belousov { 1728c6f8f3dSKonstantin Belousov u_int cp[4]; 1738c6f8f3dSKonstantin Belousov uint64_t xsave_mask_user; 1748c6f8f3dSKonstantin Belousov 1758c6f8f3dSKonstantin Belousov if ((cpu_feature2 & CPUID2_XSAVE) != 0) { 1768c6f8f3dSKonstantin Belousov use_xsave = 1; 1778c6f8f3dSKonstantin Belousov TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); 1788c6f8f3dSKonstantin Belousov } 1798c6f8f3dSKonstantin Belousov if (!use_xsave) 1808c6f8f3dSKonstantin Belousov return; 1818c6f8f3dSKonstantin Belousov 1828c6f8f3dSKonstantin Belousov cpuid_count(0xd, 0x0, cp); 1838c6f8f3dSKonstantin Belousov xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; 1848c6f8f3dSKonstantin Belousov if ((cp[0] & xsave_mask) != xsave_mask) 1858c6f8f3dSKonstantin Belousov panic("CPU0 does not support X87 or SSE: %x", cp[0]); 1868c6f8f3dSKonstantin Belousov xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; 1878c6f8f3dSKonstantin Belousov xsave_mask_user = xsave_mask; 1888c6f8f3dSKonstantin Belousov TUNABLE_ULONG_FETCH("hw.xsave_mask", &xsave_mask_user); 1898c6f8f3dSKonstantin Belousov xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; 1908c6f8f3dSKonstantin Belousov xsave_mask &= xsave_mask_user; 191333d0c60SKonstantin Belousov 192333d0c60SKonstantin Belousov cpuid_count(0xd, 0x1, cp); 193333d0c60SKonstantin Belousov if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) { 194333d0c60SKonstantin Belousov /* 195333d0c60SKonstantin Belousov * Patch the XSAVE instruction in the cpu_switch code 196333d0c60SKonstantin Belousov * to XSAVEOPT. We assume that XSAVE encoding used 197333d0c60SKonstantin Belousov * REX byte, and set the bit 4 of the r/m byte. 198333d0c60SKonstantin Belousov */ 199333d0c60SKonstantin Belousov ctx_switch_xsave[3] |= 0x10; 200333d0c60SKonstantin Belousov } 2018c6f8f3dSKonstantin Belousov } 2028c6f8f3dSKonstantin Belousov 2038c6f8f3dSKonstantin Belousov /* 2048c6f8f3dSKonstantin Belousov * Calculate the fpu save area size. 2058c6f8f3dSKonstantin Belousov */ 2068c6f8f3dSKonstantin Belousov static void 2078c6f8f3dSKonstantin Belousov fpuinit_bsp2(void) 2088c6f8f3dSKonstantin Belousov { 2098c6f8f3dSKonstantin Belousov u_int cp[4]; 2108c6f8f3dSKonstantin Belousov 2118c6f8f3dSKonstantin Belousov if (use_xsave) { 2128c6f8f3dSKonstantin Belousov cpuid_count(0xd, 0x0, cp); 2138c6f8f3dSKonstantin Belousov cpu_max_ext_state_size = cp[1]; 2148c6f8f3dSKonstantin Belousov 2158c6f8f3dSKonstantin Belousov /* 2168c6f8f3dSKonstantin Belousov * Reload the cpu_feature2, since we enabled OSXSAVE. 2178c6f8f3dSKonstantin Belousov */ 2188c6f8f3dSKonstantin Belousov do_cpuid(1, cp); 2198c6f8f3dSKonstantin Belousov cpu_feature2 = cp[2]; 2208c6f8f3dSKonstantin Belousov } else 2218c6f8f3dSKonstantin Belousov cpu_max_ext_state_size = sizeof(struct savefpu); 2228c6f8f3dSKonstantin Belousov } 2238c6f8f3dSKonstantin Belousov 2248c6f8f3dSKonstantin Belousov /* 2258c6f8f3dSKonstantin Belousov * Initialize the floating point unit. 226da4113b3SPeter Wemm */ 227398dbb11SPeter Wemm void 2281c89210cSPeter Wemm fpuinit(void) 229da4113b3SPeter Wemm { 2300689bdccSJohn Baldwin register_t saveintr; 23196a7759eSPeter Wemm u_int mxcsr; 232398dbb11SPeter Wemm u_short control; 233da4113b3SPeter Wemm 2348c6f8f3dSKonstantin Belousov if (IS_BSP()) 2358c6f8f3dSKonstantin Belousov fpuinit_bsp1(); 2368c6f8f3dSKonstantin Belousov 2378c6f8f3dSKonstantin Belousov if (use_xsave) { 2388c6f8f3dSKonstantin Belousov load_cr4(rcr4() | CR4_XSAVE); 2397574a595SJohn Baldwin load_xcr(XCR0, xsave_mask); 2408c6f8f3dSKonstantin Belousov } 2418c6f8f3dSKonstantin Belousov 2428c6f8f3dSKonstantin Belousov /* 2438c6f8f3dSKonstantin Belousov * XCR0 shall be set up before CPU can report the save area size. 2448c6f8f3dSKonstantin Belousov */ 2458c6f8f3dSKonstantin Belousov if (IS_BSP()) 2468c6f8f3dSKonstantin Belousov fpuinit_bsp2(); 2478c6f8f3dSKonstantin Belousov 24899753495SKonstantin Belousov /* 24999753495SKonstantin Belousov * It is too early for critical_enter() to work on AP. 25099753495SKonstantin Belousov */ 2510689bdccSJohn Baldwin saveintr = intr_disable(); 2525b81b6b3SRodney W. Grimes stop_emulating(); 2535b81b6b3SRodney W. Grimes fninit(); 254398dbb11SPeter Wemm control = __INITIAL_FPUCW__; 25517275403SJung-uk Kim fldcw(control); 25696a7759eSPeter Wemm mxcsr = __INITIAL_MXCSR__; 25796a7759eSPeter Wemm ldmxcsr(mxcsr); 258a8346a98SJohn Baldwin start_emulating(); 2590689bdccSJohn Baldwin intr_restore(saveintr); 2605b81b6b3SRodney W. Grimes } 2615b81b6b3SRodney W. Grimes 2625b81b6b3SRodney W. Grimes /* 2638c6f8f3dSKonstantin Belousov * On the boot CPU we generate a clean state that is used to 2648c6f8f3dSKonstantin Belousov * initialize the floating point unit when it is first used by a 2658c6f8f3dSKonstantin Belousov * process. 2668c6f8f3dSKonstantin Belousov */ 2678c6f8f3dSKonstantin Belousov static void 2688c6f8f3dSKonstantin Belousov fpuinitstate(void *arg __unused) 2698c6f8f3dSKonstantin Belousov { 2708c6f8f3dSKonstantin Belousov register_t saveintr; 271333d0c60SKonstantin Belousov int cp[4], i, max_ext_n; 2728c6f8f3dSKonstantin Belousov 2738c6f8f3dSKonstantin Belousov fpu_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF, 2748c6f8f3dSKonstantin Belousov M_WAITOK | M_ZERO); 2758c6f8f3dSKonstantin Belousov saveintr = intr_disable(); 2768c6f8f3dSKonstantin Belousov stop_emulating(); 2778c6f8f3dSKonstantin Belousov 2788c6f8f3dSKonstantin Belousov fpusave(fpu_initialstate); 2798c6f8f3dSKonstantin Belousov if (fpu_initialstate->sv_env.en_mxcsr_mask) 2808c6f8f3dSKonstantin Belousov cpu_mxcsr_mask = fpu_initialstate->sv_env.en_mxcsr_mask; 2818c6f8f3dSKonstantin Belousov else 2828c6f8f3dSKonstantin Belousov cpu_mxcsr_mask = 0xFFBF; 2838c6f8f3dSKonstantin Belousov 2848c6f8f3dSKonstantin Belousov /* 2858c6f8f3dSKonstantin Belousov * The fninit instruction does not modify XMM registers. The 2868c6f8f3dSKonstantin Belousov * fpusave call dumped the garbage contained in the registers 2878c6f8f3dSKonstantin Belousov * after reset to the initial state saved. Clear XMM 2888c6f8f3dSKonstantin Belousov * registers file image to make the startup program state and 2898c6f8f3dSKonstantin Belousov * signal handler XMM register content predictable. 2908c6f8f3dSKonstantin Belousov */ 2918c6f8f3dSKonstantin Belousov bzero(&fpu_initialstate->sv_xmm[0], sizeof(struct xmmacc)); 2928c6f8f3dSKonstantin Belousov 293333d0c60SKonstantin Belousov /* 294333d0c60SKonstantin Belousov * Create a table describing the layout of the CPU Extended 295333d0c60SKonstantin Belousov * Save Area. 296333d0c60SKonstantin Belousov */ 29714f52559SKonstantin Belousov if (use_xsave) { 298333d0c60SKonstantin Belousov max_ext_n = flsl(xsave_mask); 299333d0c60SKonstantin Belousov xsave_area_desc = malloc(max_ext_n * sizeof(struct 300333d0c60SKonstantin Belousov xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO); 301333d0c60SKonstantin Belousov /* x87 state */ 302333d0c60SKonstantin Belousov xsave_area_desc[0].offset = 0; 303333d0c60SKonstantin Belousov xsave_area_desc[0].size = 160; 304333d0c60SKonstantin Belousov /* XMM */ 305333d0c60SKonstantin Belousov xsave_area_desc[1].offset = 160; 306333d0c60SKonstantin Belousov xsave_area_desc[1].size = 288 - 160; 307333d0c60SKonstantin Belousov 308333d0c60SKonstantin Belousov for (i = 2; i < max_ext_n; i++) { 309333d0c60SKonstantin Belousov cpuid_count(0xd, i, cp); 310333d0c60SKonstantin Belousov xsave_area_desc[i].offset = cp[1]; 311333d0c60SKonstantin Belousov xsave_area_desc[i].size = cp[0]; 312333d0c60SKonstantin Belousov } 313333d0c60SKonstantin Belousov } 314333d0c60SKonstantin Belousov 3152741efecSPeter Grehan fpu_save_area_zone = uma_zcreate("FPU_save_area", 3162741efecSPeter Grehan cpu_max_ext_state_size, NULL, NULL, NULL, NULL, 3172741efecSPeter Grehan XSAVE_AREA_ALIGN - 1, 0); 3182741efecSPeter Grehan 3198c6f8f3dSKonstantin Belousov start_emulating(); 3208c6f8f3dSKonstantin Belousov intr_restore(saveintr); 3218c6f8f3dSKonstantin Belousov } 3228c6f8f3dSKonstantin Belousov SYSINIT(fpuinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, fpuinitstate, NULL); 3238c6f8f3dSKonstantin Belousov 3248c6f8f3dSKonstantin Belousov /* 3255b81b6b3SRodney W. Grimes * Free coprocessor (if we have it). 3265b81b6b3SRodney W. Grimes */ 3275b81b6b3SRodney W. Grimes void 328bf2f09eeSPeter Wemm fpuexit(struct thread *td) 3295b81b6b3SRodney W. Grimes { 3305b81b6b3SRodney W. Grimes 33199753495SKonstantin Belousov critical_enter(); 3321c89210cSPeter Wemm if (curthread == PCPU_GET(fpcurthread)) { 3331c89210cSPeter Wemm stop_emulating(); 33483b22b05SKonstantin Belousov fpusave(curpcb->pcb_save); 3351c89210cSPeter Wemm start_emulating(); 3361c89210cSPeter Wemm PCPU_SET(fpcurthread, 0); 3371c89210cSPeter Wemm } 33899753495SKonstantin Belousov critical_exit(); 3395b81b6b3SRodney W. Grimes } 3405b81b6b3SRodney W. Grimes 34130abe507SJonathan Mini int 342bf2f09eeSPeter Wemm fpuformat() 34330abe507SJonathan Mini { 34430abe507SJonathan Mini 34530abe507SJonathan Mini return (_MC_FPFMT_XMM); 34630abe507SJonathan Mini } 34730abe507SJonathan Mini 3485b81b6b3SRodney W. Grimes /* 349a7674320SMartin Cracauer * The following mechanism is used to ensure that the FPE_... value 350a7674320SMartin Cracauer * that is passed as a trapcode to the signal handler of the user 351a7674320SMartin Cracauer * process does not have more than one bit set. 352a7674320SMartin Cracauer * 353a7674320SMartin Cracauer * Multiple bits may be set if the user process modifies the control 354a7674320SMartin Cracauer * word while a status word bit is already set. While this is a sign 355a7674320SMartin Cracauer * of bad coding, we have no choise than to narrow them down to one 356a7674320SMartin Cracauer * bit, since we must not send a trapcode that is not exactly one of 357a7674320SMartin Cracauer * the FPE_ macros. 358a7674320SMartin Cracauer * 359a7674320SMartin Cracauer * The mechanism has a static table with 127 entries. Each combination 360a7674320SMartin Cracauer * of the 7 FPU status word exception bits directly translates to a 361a7674320SMartin Cracauer * position in this table, where a single FPE_... value is stored. 362a7674320SMartin Cracauer * This FPE_... value stored there is considered the "most important" 363a7674320SMartin Cracauer * of the exception bits and will be sent as the signal code. The 364a7674320SMartin Cracauer * precedence of the bits is based upon Intel Document "Numerical 365a7674320SMartin Cracauer * Applications", Chapter "Special Computational Situations". 366a7674320SMartin Cracauer * 367a7674320SMartin Cracauer * The macro to choose one of these values does these steps: 1) Throw 368a7674320SMartin Cracauer * away status word bits that cannot be masked. 2) Throw away the bits 369a7674320SMartin Cracauer * currently masked in the control word, assuming the user isn't 370a7674320SMartin Cracauer * interested in them anymore. 3) Reinsert status word bit 7 (stack 371a7674320SMartin Cracauer * fault) if it is set, which cannot be masked but must be presered. 372a7674320SMartin Cracauer * 4) Use the remaining bits to point into the trapcode table. 373a7674320SMartin Cracauer * 374a7674320SMartin Cracauer * The 6 maskable bits in order of their preference, as stated in the 375a7674320SMartin Cracauer * above referenced Intel manual: 376a7674320SMartin Cracauer * 1 Invalid operation (FP_X_INV) 377a7674320SMartin Cracauer * 1a Stack underflow 378a7674320SMartin Cracauer * 1b Stack overflow 379a7674320SMartin Cracauer * 1c Operand of unsupported format 380a7674320SMartin Cracauer * 1d SNaN operand. 381a7674320SMartin Cracauer * 2 QNaN operand (not an exception, irrelavant here) 382a7674320SMartin Cracauer * 3 Any other invalid-operation not mentioned above or zero divide 383a7674320SMartin Cracauer * (FP_X_INV, FP_X_DZ) 384a7674320SMartin Cracauer * 4 Denormal operand (FP_X_DNML) 385a7674320SMartin Cracauer * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) 386784648c6SMartin Cracauer * 6 Inexact result (FP_X_IMP) 387784648c6SMartin Cracauer */ 388a7674320SMartin Cracauer static char fpetable[128] = { 389a7674320SMartin Cracauer 0, 390a7674320SMartin Cracauer FPE_FLTINV, /* 1 - INV */ 391a7674320SMartin Cracauer FPE_FLTUND, /* 2 - DNML */ 392a7674320SMartin Cracauer FPE_FLTINV, /* 3 - INV | DNML */ 393a7674320SMartin Cracauer FPE_FLTDIV, /* 4 - DZ */ 394a7674320SMartin Cracauer FPE_FLTINV, /* 5 - INV | DZ */ 395a7674320SMartin Cracauer FPE_FLTDIV, /* 6 - DNML | DZ */ 396a7674320SMartin Cracauer FPE_FLTINV, /* 7 - INV | DNML | DZ */ 397a7674320SMartin Cracauer FPE_FLTOVF, /* 8 - OFL */ 398a7674320SMartin Cracauer FPE_FLTINV, /* 9 - INV | OFL */ 399a7674320SMartin Cracauer FPE_FLTUND, /* A - DNML | OFL */ 400a7674320SMartin Cracauer FPE_FLTINV, /* B - INV | DNML | OFL */ 401a7674320SMartin Cracauer FPE_FLTDIV, /* C - DZ | OFL */ 402a7674320SMartin Cracauer FPE_FLTINV, /* D - INV | DZ | OFL */ 403a7674320SMartin Cracauer FPE_FLTDIV, /* E - DNML | DZ | OFL */ 404a7674320SMartin Cracauer FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ 405a7674320SMartin Cracauer FPE_FLTUND, /* 10 - UFL */ 406a7674320SMartin Cracauer FPE_FLTINV, /* 11 - INV | UFL */ 407a7674320SMartin Cracauer FPE_FLTUND, /* 12 - DNML | UFL */ 408a7674320SMartin Cracauer FPE_FLTINV, /* 13 - INV | DNML | UFL */ 409a7674320SMartin Cracauer FPE_FLTDIV, /* 14 - DZ | UFL */ 410a7674320SMartin Cracauer FPE_FLTINV, /* 15 - INV | DZ | UFL */ 411a7674320SMartin Cracauer FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ 412a7674320SMartin Cracauer FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ 413a7674320SMartin Cracauer FPE_FLTOVF, /* 18 - OFL | UFL */ 414a7674320SMartin Cracauer FPE_FLTINV, /* 19 - INV | OFL | UFL */ 415a7674320SMartin Cracauer FPE_FLTUND, /* 1A - DNML | OFL | UFL */ 416a7674320SMartin Cracauer FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ 417a7674320SMartin Cracauer FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ 418a7674320SMartin Cracauer FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ 419a7674320SMartin Cracauer FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ 420a7674320SMartin Cracauer FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ 421a7674320SMartin Cracauer FPE_FLTRES, /* 20 - IMP */ 422a7674320SMartin Cracauer FPE_FLTINV, /* 21 - INV | IMP */ 423a7674320SMartin Cracauer FPE_FLTUND, /* 22 - DNML | IMP */ 424a7674320SMartin Cracauer FPE_FLTINV, /* 23 - INV | DNML | IMP */ 425a7674320SMartin Cracauer FPE_FLTDIV, /* 24 - DZ | IMP */ 426a7674320SMartin Cracauer FPE_FLTINV, /* 25 - INV | DZ | IMP */ 427a7674320SMartin Cracauer FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ 428a7674320SMartin Cracauer FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ 429a7674320SMartin Cracauer FPE_FLTOVF, /* 28 - OFL | IMP */ 430a7674320SMartin Cracauer FPE_FLTINV, /* 29 - INV | OFL | IMP */ 431a7674320SMartin Cracauer FPE_FLTUND, /* 2A - DNML | OFL | IMP */ 432a7674320SMartin Cracauer FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ 433a7674320SMartin Cracauer FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ 434a7674320SMartin Cracauer FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ 435a7674320SMartin Cracauer FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ 436a7674320SMartin Cracauer FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ 437a7674320SMartin Cracauer FPE_FLTUND, /* 30 - UFL | IMP */ 438a7674320SMartin Cracauer FPE_FLTINV, /* 31 - INV | UFL | IMP */ 439a7674320SMartin Cracauer FPE_FLTUND, /* 32 - DNML | UFL | IMP */ 440a7674320SMartin Cracauer FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ 441a7674320SMartin Cracauer FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ 442a7674320SMartin Cracauer FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ 443a7674320SMartin Cracauer FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ 444a7674320SMartin Cracauer FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ 445a7674320SMartin Cracauer FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ 446a7674320SMartin Cracauer FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ 447a7674320SMartin Cracauer FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ 448a7674320SMartin Cracauer FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ 449a7674320SMartin Cracauer FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ 450a7674320SMartin Cracauer FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ 451a7674320SMartin Cracauer FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ 452a7674320SMartin Cracauer FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ 453a7674320SMartin Cracauer FPE_FLTSUB, /* 40 - STK */ 454a7674320SMartin Cracauer FPE_FLTSUB, /* 41 - INV | STK */ 455a7674320SMartin Cracauer FPE_FLTUND, /* 42 - DNML | STK */ 456a7674320SMartin Cracauer FPE_FLTSUB, /* 43 - INV | DNML | STK */ 457a7674320SMartin Cracauer FPE_FLTDIV, /* 44 - DZ | STK */ 458a7674320SMartin Cracauer FPE_FLTSUB, /* 45 - INV | DZ | STK */ 459a7674320SMartin Cracauer FPE_FLTDIV, /* 46 - DNML | DZ | STK */ 460a7674320SMartin Cracauer FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ 461a7674320SMartin Cracauer FPE_FLTOVF, /* 48 - OFL | STK */ 462a7674320SMartin Cracauer FPE_FLTSUB, /* 49 - INV | OFL | STK */ 463a7674320SMartin Cracauer FPE_FLTUND, /* 4A - DNML | OFL | STK */ 464a7674320SMartin Cracauer FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ 465a7674320SMartin Cracauer FPE_FLTDIV, /* 4C - DZ | OFL | STK */ 466a7674320SMartin Cracauer FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ 467a7674320SMartin Cracauer FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ 468a7674320SMartin Cracauer FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ 469a7674320SMartin Cracauer FPE_FLTUND, /* 50 - UFL | STK */ 470a7674320SMartin Cracauer FPE_FLTSUB, /* 51 - INV | UFL | STK */ 471a7674320SMartin Cracauer FPE_FLTUND, /* 52 - DNML | UFL | STK */ 472a7674320SMartin Cracauer FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ 473a7674320SMartin Cracauer FPE_FLTDIV, /* 54 - DZ | UFL | STK */ 474a7674320SMartin Cracauer FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ 475a7674320SMartin Cracauer FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ 476a7674320SMartin Cracauer FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ 477a7674320SMartin Cracauer FPE_FLTOVF, /* 58 - OFL | UFL | STK */ 478a7674320SMartin Cracauer FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ 479a7674320SMartin Cracauer FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ 480a7674320SMartin Cracauer FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ 481a7674320SMartin Cracauer FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ 482a7674320SMartin Cracauer FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ 483a7674320SMartin Cracauer FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ 484a7674320SMartin Cracauer FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ 485a7674320SMartin Cracauer FPE_FLTRES, /* 60 - IMP | STK */ 486a7674320SMartin Cracauer FPE_FLTSUB, /* 61 - INV | IMP | STK */ 487a7674320SMartin Cracauer FPE_FLTUND, /* 62 - DNML | IMP | STK */ 488a7674320SMartin Cracauer FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ 489a7674320SMartin Cracauer FPE_FLTDIV, /* 64 - DZ | IMP | STK */ 490a7674320SMartin Cracauer FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ 491a7674320SMartin Cracauer FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ 492a7674320SMartin Cracauer FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ 493a7674320SMartin Cracauer FPE_FLTOVF, /* 68 - OFL | IMP | STK */ 494a7674320SMartin Cracauer FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ 495a7674320SMartin Cracauer FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ 496a7674320SMartin Cracauer FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ 497a7674320SMartin Cracauer FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ 498a7674320SMartin Cracauer FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ 499a7674320SMartin Cracauer FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ 500a7674320SMartin Cracauer FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ 501a7674320SMartin Cracauer FPE_FLTUND, /* 70 - UFL | IMP | STK */ 502a7674320SMartin Cracauer FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ 503a7674320SMartin Cracauer FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ 504a7674320SMartin Cracauer FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ 505a7674320SMartin Cracauer FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ 506a7674320SMartin Cracauer FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ 507a7674320SMartin Cracauer FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ 508a7674320SMartin Cracauer FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ 509a7674320SMartin Cracauer FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ 510a7674320SMartin Cracauer FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ 511a7674320SMartin Cracauer FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ 512a7674320SMartin Cracauer FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ 513a7674320SMartin Cracauer FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ 514a7674320SMartin Cracauer FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ 515a7674320SMartin Cracauer FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ 516a7674320SMartin Cracauer FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ 517a7674320SMartin Cracauer }; 518a7674320SMartin Cracauer 519a7674320SMartin Cracauer /* 520dfa8a512SKonstantin Belousov * Read the FP status and control words, then generate si_code value 521dfa8a512SKonstantin Belousov * for SIGFPE. The error code chosen will be one of the 522dfa8a512SKonstantin Belousov * FPE_... macros. It will be sent as the second argument to old 523dfa8a512SKonstantin Belousov * BSD-style signal handlers and as "siginfo_t->si_code" (second 524dfa8a512SKonstantin Belousov * argument) to SA_SIGINFO signal handlers. 5255b81b6b3SRodney W. Grimes * 526dfa8a512SKonstantin Belousov * Some time ago, we cleared the x87 exceptions with FNCLEX there. 527dfa8a512SKonstantin Belousov * Clearing exceptions was necessary mainly to avoid IRQ13 bugs. The 528dfa8a512SKonstantin Belousov * usermode code which understands the FPU hardware enough to enable 529dfa8a512SKonstantin Belousov * the exceptions, can also handle clearing the exception state in the 530dfa8a512SKonstantin Belousov * handler. The only consequence of not clearing the exception is the 531dfa8a512SKonstantin Belousov * rethrow of the SIGFPE on return from the signal handler and 532dfa8a512SKonstantin Belousov * reexecution of the corresponding instruction. 533bc84db62SKonstantin Belousov * 534dfa8a512SKonstantin Belousov * For XMM traps, the exceptions were never cleared. 5355b81b6b3SRodney W. Grimes */ 5361c1771cbSBruce Evans int 537bc84db62SKonstantin Belousov fputrap_x87(void) 5385b81b6b3SRodney W. Grimes { 539bc84db62SKonstantin Belousov struct savefpu *pcb_save; 5401c1771cbSBruce Evans u_short control, status; 5415b81b6b3SRodney W. Grimes 54299753495SKonstantin Belousov critical_enter(); 5435b81b6b3SRodney W. Grimes 5445b81b6b3SRodney W. Grimes /* 5451c1771cbSBruce Evans * Interrupt handling (for another interrupt) may have pushed the 5461c1771cbSBruce Evans * state to memory. Fetch the relevant parts of the state from 5471c1771cbSBruce Evans * wherever they are. 5485b81b6b3SRodney W. Grimes */ 5490bbc8826SJohn Baldwin if (PCPU_GET(fpcurthread) != curthread) { 55083b22b05SKonstantin Belousov pcb_save = curpcb->pcb_save; 551bc84db62SKonstantin Belousov control = pcb_save->sv_env.en_cw; 552bc84db62SKonstantin Belousov status = pcb_save->sv_env.en_sw; 5535b81b6b3SRodney W. Grimes } else { 5541c1771cbSBruce Evans fnstcw(&control); 5551c1771cbSBruce Evans fnstsw(&status); 5565b81b6b3SRodney W. Grimes } 5571c1771cbSBruce Evans 55899753495SKonstantin Belousov critical_exit(); 5591c1771cbSBruce Evans return (fpetable[status & ((~control & 0x3f) | 0x40)]); 5605b81b6b3SRodney W. Grimes } 5615b81b6b3SRodney W. Grimes 562bc84db62SKonstantin Belousov int 563bc84db62SKonstantin Belousov fputrap_sse(void) 564bc84db62SKonstantin Belousov { 565bc84db62SKonstantin Belousov u_int mxcsr; 566bc84db62SKonstantin Belousov 567bc84db62SKonstantin Belousov critical_enter(); 568bc84db62SKonstantin Belousov if (PCPU_GET(fpcurthread) != curthread) 56983b22b05SKonstantin Belousov mxcsr = curpcb->pcb_save->sv_env.en_mxcsr; 570bc84db62SKonstantin Belousov else 571bc84db62SKonstantin Belousov stmxcsr(&mxcsr); 572bc84db62SKonstantin Belousov critical_exit(); 573bc84db62SKonstantin Belousov return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]); 574bc84db62SKonstantin Belousov } 575bc84db62SKonstantin Belousov 5765b81b6b3SRodney W. Grimes /* 5775b81b6b3SRodney W. Grimes * Implement device not available (DNA) exception 5785b81b6b3SRodney W. Grimes * 5790bbc8826SJohn Baldwin * It would be better to switch FP context here (if curthread != fpcurthread) 58037e52b59SBruce Evans * and not necessarily for every context switch, but it is too hard to 58137e52b59SBruce Evans * access foreign pcb's. 5825b81b6b3SRodney W. Grimes */ 58330abe507SJonathan Mini 58430abe507SJonathan Mini static int err_count = 0; 58530abe507SJonathan Mini 586a8346a98SJohn Baldwin void 587a8346a98SJohn Baldwin fpudna(void) 5885b81b6b3SRodney W. Grimes { 58905f6ee66SJake Burkholder 59099753495SKonstantin Belousov critical_enter(); 59130abe507SJonathan Mini if (PCPU_GET(fpcurthread) == curthread) { 592bf2f09eeSPeter Wemm printf("fpudna: fpcurthread == curthread %d times\n", 59330abe507SJonathan Mini ++err_count); 59430abe507SJonathan Mini stop_emulating(); 59599753495SKonstantin Belousov critical_exit(); 596a8346a98SJohn Baldwin return; 59730abe507SJonathan Mini } 5980bbc8826SJohn Baldwin if (PCPU_GET(fpcurthread) != NULL) { 599bf2f09eeSPeter Wemm printf("fpudna: fpcurthread = %p (%d), curthread = %p (%d)\n", 60030abe507SJonathan Mini PCPU_GET(fpcurthread), 60130abe507SJonathan Mini PCPU_GET(fpcurthread)->td_proc->p_pid, 60230abe507SJonathan Mini curthread, curthread->td_proc->p_pid); 603bf2f09eeSPeter Wemm panic("fpudna"); 6045b81b6b3SRodney W. Grimes } 6055b81b6b3SRodney W. Grimes stop_emulating(); 6065b81b6b3SRodney W. Grimes /* 607bf2f09eeSPeter Wemm * Record new context early in case frstor causes a trap. 6085b81b6b3SRodney W. Grimes */ 6090bbc8826SJohn Baldwin PCPU_SET(fpcurthread, curthread); 6109d146ac5SPeter Wemm 6112652af56SColin Percival fpu_clean_state(); 6122652af56SColin Percival 6131965c139SKonstantin Belousov if ((curpcb->pcb_flags & PCB_FPUINITDONE) == 0) { 6145b81b6b3SRodney W. Grimes /* 61563de9515SJohn Baldwin * This is the first time this thread has used the FPU or 61663de9515SJohn Baldwin * the PCB doesn't contain a clean FPU state. Explicitly 61763de9515SJohn Baldwin * load an initial state. 618333d0c60SKonstantin Belousov * 619333d0c60SKonstantin Belousov * We prefer to restore the state from the actual save 620333d0c60SKonstantin Belousov * area in PCB instead of directly loading from 621333d0c60SKonstantin Belousov * fpu_initialstate, to ignite the XSAVEOPT 622333d0c60SKonstantin Belousov * tracking engine. 6235b81b6b3SRodney W. Grimes */ 6241965c139SKonstantin Belousov bcopy(fpu_initialstate, curpcb->pcb_save, cpu_max_ext_state_size); 6251965c139SKonstantin Belousov fpurestore(curpcb->pcb_save); 6261965c139SKonstantin Belousov if (curpcb->pcb_initial_fpucw != __INITIAL_FPUCW__) 6271965c139SKonstantin Belousov fldcw(curpcb->pcb_initial_fpucw); 6281965c139SKonstantin Belousov if (PCB_USER_FPU(curpcb)) 6291965c139SKonstantin Belousov set_pcb_flags(curpcb, 630e6c006d9SJung-uk Kim PCB_FPUINITDONE | PCB_USERFPUINITDONE); 631e6c006d9SJung-uk Kim else 6321965c139SKonstantin Belousov set_pcb_flags(curpcb, PCB_FPUINITDONE); 6331c89210cSPeter Wemm } else 6341965c139SKonstantin Belousov fpurestore(curpcb->pcb_save); 63599753495SKonstantin Belousov critical_exit(); 6365b81b6b3SRodney W. Grimes } 6375b81b6b3SRodney W. Grimes 63830abe507SJonathan Mini void 639bf2f09eeSPeter Wemm fpudrop() 64030abe507SJonathan Mini { 64130abe507SJonathan Mini struct thread *td; 64230abe507SJonathan Mini 64330abe507SJonathan Mini td = PCPU_GET(fpcurthread); 64499753495SKonstantin Belousov KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); 6454a23ecc7SKonstantin Belousov CRITICAL_ASSERT(td); 64630abe507SJonathan Mini PCPU_SET(fpcurthread, NULL); 647e6c006d9SJung-uk Kim clear_pcb_flags(td->td_pcb, PCB_FPUINITDONE); 64830abe507SJonathan Mini start_emulating(); 64930abe507SJonathan Mini } 65030abe507SJonathan Mini 65130abe507SJonathan Mini /* 6525c6eb037SKonstantin Belousov * Get the user state of the FPU into pcb->pcb_user_save without 6535c6eb037SKonstantin Belousov * dropping ownership (if possible). It returns the FPU ownership 6545c6eb037SKonstantin Belousov * status. 65530abe507SJonathan Mini */ 65630abe507SJonathan Mini int 6575c6eb037SKonstantin Belousov fpugetregs(struct thread *td) 6586cf9a08dSKonstantin Belousov { 6596cf9a08dSKonstantin Belousov struct pcb *pcb; 660333d0c60SKonstantin Belousov uint64_t *xstate_bv, bit; 661333d0c60SKonstantin Belousov char *sa; 66214f52559SKonstantin Belousov int max_ext_n, i, owned; 6636cf9a08dSKonstantin Belousov 6646cf9a08dSKonstantin Belousov pcb = td->td_pcb; 6656cf9a08dSKonstantin Belousov if ((pcb->pcb_flags & PCB_USERFPUINITDONE) == 0) { 6668c6f8f3dSKonstantin Belousov bcopy(fpu_initialstate, get_pcb_user_save_pcb(pcb), 6678c6f8f3dSKonstantin Belousov cpu_max_ext_state_size); 6688c6f8f3dSKonstantin Belousov get_pcb_user_save_pcb(pcb)->sv_env.en_cw = 6698c6f8f3dSKonstantin Belousov pcb->pcb_initial_fpucw; 6705c6eb037SKonstantin Belousov fpuuserinited(td); 6715c6eb037SKonstantin Belousov return (_MC_FPOWNED_PCB); 6726cf9a08dSKonstantin Belousov } 67399753495SKonstantin Belousov critical_enter(); 6746cf9a08dSKonstantin Belousov if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 6758c6f8f3dSKonstantin Belousov fpusave(get_pcb_user_save_pcb(pcb)); 67614f52559SKonstantin Belousov owned = _MC_FPOWNED_FPU; 6776cf9a08dSKonstantin Belousov } else { 67814f52559SKonstantin Belousov owned = _MC_FPOWNED_PCB; 67914f52559SKonstantin Belousov } 68099753495SKonstantin Belousov critical_exit(); 68114f52559SKonstantin Belousov if (use_xsave) { 682333d0c60SKonstantin Belousov /* 683333d0c60SKonstantin Belousov * Handle partially saved state. 684333d0c60SKonstantin Belousov */ 685333d0c60SKonstantin Belousov sa = (char *)get_pcb_user_save_pcb(pcb); 686333d0c60SKonstantin Belousov xstate_bv = (uint64_t *)(sa + sizeof(struct savefpu) + 687333d0c60SKonstantin Belousov offsetof(struct xstate_hdr, xstate_bv)); 688333d0c60SKonstantin Belousov max_ext_n = flsl(xsave_mask); 689333d0c60SKonstantin Belousov for (i = 0; i < max_ext_n; i++) { 690*241b67bbSKonstantin Belousov bit = 1ULL << i; 691*241b67bbSKonstantin Belousov if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0) 692333d0c60SKonstantin Belousov continue; 693333d0c60SKonstantin Belousov bcopy((char *)fpu_initialstate + 694333d0c60SKonstantin Belousov xsave_area_desc[i].offset, 695333d0c60SKonstantin Belousov sa + xsave_area_desc[i].offset, 696333d0c60SKonstantin Belousov xsave_area_desc[i].size); 697333d0c60SKonstantin Belousov *xstate_bv |= bit; 698333d0c60SKonstantin Belousov } 699333d0c60SKonstantin Belousov } 70014f52559SKonstantin Belousov return (owned); 7016cf9a08dSKonstantin Belousov } 7026cf9a08dSKonstantin Belousov 7035c6eb037SKonstantin Belousov void 7045c6eb037SKonstantin Belousov fpuuserinited(struct thread *td) 70530abe507SJonathan Mini { 7066cf9a08dSKonstantin Belousov struct pcb *pcb; 70730abe507SJonathan Mini 7086cf9a08dSKonstantin Belousov pcb = td->td_pcb; 7095c6eb037SKonstantin Belousov if (PCB_USER_FPU(pcb)) 710e6c006d9SJung-uk Kim set_pcb_flags(pcb, 711e6c006d9SJung-uk Kim PCB_FPUINITDONE | PCB_USERFPUINITDONE); 712e6c006d9SJung-uk Kim else 713e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 71430abe507SJonathan Mini } 71530abe507SJonathan Mini 7168c6f8f3dSKonstantin Belousov int 7178c6f8f3dSKonstantin Belousov fpusetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size) 7188c6f8f3dSKonstantin Belousov { 7198c6f8f3dSKonstantin Belousov struct xstate_hdr *hdr, *ehdr; 7208c6f8f3dSKonstantin Belousov size_t len, max_len; 7218c6f8f3dSKonstantin Belousov uint64_t bv; 7228c6f8f3dSKonstantin Belousov 7238c6f8f3dSKonstantin Belousov /* XXXKIB should we clear all extended state in xstate_bv instead ? */ 7248c6f8f3dSKonstantin Belousov if (xfpustate == NULL) 7258c6f8f3dSKonstantin Belousov return (0); 7268c6f8f3dSKonstantin Belousov if (!use_xsave) 7278c6f8f3dSKonstantin Belousov return (EOPNOTSUPP); 7288c6f8f3dSKonstantin Belousov 7298c6f8f3dSKonstantin Belousov len = xfpustate_size; 7308c6f8f3dSKonstantin Belousov if (len < sizeof(struct xstate_hdr)) 7318c6f8f3dSKonstantin Belousov return (EINVAL); 7328c6f8f3dSKonstantin Belousov max_len = cpu_max_ext_state_size - sizeof(struct savefpu); 7338c6f8f3dSKonstantin Belousov if (len > max_len) 7348c6f8f3dSKonstantin Belousov return (EINVAL); 7358c6f8f3dSKonstantin Belousov 7368c6f8f3dSKonstantin Belousov ehdr = (struct xstate_hdr *)xfpustate; 7378c6f8f3dSKonstantin Belousov bv = ehdr->xstate_bv; 7388c6f8f3dSKonstantin Belousov 7398c6f8f3dSKonstantin Belousov /* 7408c6f8f3dSKonstantin Belousov * Avoid #gp. 7418c6f8f3dSKonstantin Belousov */ 7428c6f8f3dSKonstantin Belousov if (bv & ~xsave_mask) 7438c6f8f3dSKonstantin Belousov return (EINVAL); 7448c6f8f3dSKonstantin Belousov 7458c6f8f3dSKonstantin Belousov hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1); 7468c6f8f3dSKonstantin Belousov 7478c6f8f3dSKonstantin Belousov hdr->xstate_bv = bv; 7488c6f8f3dSKonstantin Belousov bcopy(xfpustate + sizeof(struct xstate_hdr), 7498c6f8f3dSKonstantin Belousov (char *)(hdr + 1), len - sizeof(struct xstate_hdr)); 7508c6f8f3dSKonstantin Belousov 7518c6f8f3dSKonstantin Belousov return (0); 7528c6f8f3dSKonstantin Belousov } 7538c6f8f3dSKonstantin Belousov 75430abe507SJonathan Mini /* 75530abe507SJonathan Mini * Set the state of the FPU. 75630abe507SJonathan Mini */ 7578c6f8f3dSKonstantin Belousov int 7588c6f8f3dSKonstantin Belousov fpusetregs(struct thread *td, struct savefpu *addr, char *xfpustate, 7598c6f8f3dSKonstantin Belousov size_t xfpustate_size) 7606cf9a08dSKonstantin Belousov { 7616cf9a08dSKonstantin Belousov struct pcb *pcb; 7628c6f8f3dSKonstantin Belousov int error; 7636cf9a08dSKonstantin Belousov 7646cf9a08dSKonstantin Belousov pcb = td->td_pcb; 76599753495SKonstantin Belousov critical_enter(); 7666cf9a08dSKonstantin Belousov if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 7678c6f8f3dSKonstantin Belousov error = fpusetxstate(td, xfpustate, xfpustate_size); 7688c6f8f3dSKonstantin Belousov if (error != 0) { 7698c6f8f3dSKonstantin Belousov critical_exit(); 7708c6f8f3dSKonstantin Belousov return (error); 7718c6f8f3dSKonstantin Belousov } 7728c6f8f3dSKonstantin Belousov bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); 7738c6f8f3dSKonstantin Belousov fpurestore(get_pcb_user_save_td(td)); 77499753495SKonstantin Belousov critical_exit(); 775e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE | PCB_USERFPUINITDONE); 7766cf9a08dSKonstantin Belousov } else { 77799753495SKonstantin Belousov critical_exit(); 7788c6f8f3dSKonstantin Belousov error = fpusetxstate(td, xfpustate, xfpustate_size); 7798c6f8f3dSKonstantin Belousov if (error != 0) 7808c6f8f3dSKonstantin Belousov return (error); 7818c6f8f3dSKonstantin Belousov bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); 7825c6eb037SKonstantin Belousov fpuuserinited(td); 7836cf9a08dSKonstantin Belousov } 7848c6f8f3dSKonstantin Belousov return (0); 7856cf9a08dSKonstantin Belousov } 7866cf9a08dSKonstantin Belousov 7876182fdbdSPeter Wemm /* 7882652af56SColin Percival * On AuthenticAMD processors, the fxrstor instruction does not restore 7892652af56SColin Percival * the x87's stored last instruction pointer, last data pointer, and last 7902652af56SColin Percival * opcode values, except in the rare case in which the exception summary 7912652af56SColin Percival * (ES) bit in the x87 status word is set to 1. 7922652af56SColin Percival * 7932652af56SColin Percival * In order to avoid leaking this information across processes, we clean 7942652af56SColin Percival * these values by performing a dummy load before executing fxrstor(). 7952652af56SColin Percival */ 7962652af56SColin Percival static void 7972652af56SColin Percival fpu_clean_state(void) 7982652af56SColin Percival { 799b9dda9d6SJohn Baldwin static float dummy_variable = 0.0; 8002652af56SColin Percival u_short status; 8012652af56SColin Percival 8022652af56SColin Percival /* 8032652af56SColin Percival * Clear the ES bit in the x87 status word if it is currently 8042652af56SColin Percival * set, in order to avoid causing a fault in the upcoming load. 8052652af56SColin Percival */ 8062652af56SColin Percival fnstsw(&status); 8072652af56SColin Percival if (status & 0x80) 8082652af56SColin Percival fnclex(); 8092652af56SColin Percival 8102652af56SColin Percival /* 8112652af56SColin Percival * Load the dummy variable into the x87 stack. This mangles 8122652af56SColin Percival * the x87 stack, but we don't care since we're about to call 8132652af56SColin Percival * fxrstor() anyway. 8142652af56SColin Percival */ 81514965052SDimitry Andric __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable)); 8162652af56SColin Percival } 8172652af56SColin Percival 8182652af56SColin Percival /* 819398dbb11SPeter Wemm * This really sucks. We want the acpi version only, but it requires 820398dbb11SPeter Wemm * the isa_if.h file in order to get the definitions. 8216182fdbdSPeter Wemm */ 822398dbb11SPeter Wemm #include "opt_isa.h" 823afa88623SPeter Wemm #ifdef DEV_ISA 824398dbb11SPeter Wemm #include <isa/isavar.h> 82554f1d0ceSGarrett Wollman /* 8265f063c7bSMike Smith * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI. 82754f1d0ceSGarrett Wollman */ 828398dbb11SPeter Wemm static struct isa_pnp_id fpupnp_ids[] = { 82954f1d0ceSGarrett Wollman { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */ 83054f1d0ceSGarrett Wollman { 0 } 83154f1d0ceSGarrett Wollman }; 83254f1d0ceSGarrett Wollman 83354f1d0ceSGarrett Wollman static int 834398dbb11SPeter Wemm fpupnp_probe(device_t dev) 83554f1d0ceSGarrett Wollman { 836bb9c06c1SMike Smith int result; 837bf2f09eeSPeter Wemm 838398dbb11SPeter Wemm result = ISA_PNP_PROBE(device_get_parent(dev), dev, fpupnp_ids); 839bf2f09eeSPeter Wemm if (result <= 0) 840bb9c06c1SMike Smith device_quiet(dev); 841bb9c06c1SMike Smith return (result); 84254f1d0ceSGarrett Wollman } 84354f1d0ceSGarrett Wollman 84454f1d0ceSGarrett Wollman static int 845398dbb11SPeter Wemm fpupnp_attach(device_t dev) 84654f1d0ceSGarrett Wollman { 847bf2f09eeSPeter Wemm 84854f1d0ceSGarrett Wollman return (0); 84954f1d0ceSGarrett Wollman } 85054f1d0ceSGarrett Wollman 851398dbb11SPeter Wemm static device_method_t fpupnp_methods[] = { 85254f1d0ceSGarrett Wollman /* Device interface */ 853398dbb11SPeter Wemm DEVMETHOD(device_probe, fpupnp_probe), 854398dbb11SPeter Wemm DEVMETHOD(device_attach, fpupnp_attach), 85554f1d0ceSGarrett Wollman DEVMETHOD(device_detach, bus_generic_detach), 85654f1d0ceSGarrett Wollman DEVMETHOD(device_shutdown, bus_generic_shutdown), 85754f1d0ceSGarrett Wollman DEVMETHOD(device_suspend, bus_generic_suspend), 85854f1d0ceSGarrett Wollman DEVMETHOD(device_resume, bus_generic_resume), 85954f1d0ceSGarrett Wollman 86054f1d0ceSGarrett Wollman { 0, 0 } 86154f1d0ceSGarrett Wollman }; 86254f1d0ceSGarrett Wollman 863398dbb11SPeter Wemm static driver_t fpupnp_driver = { 864398dbb11SPeter Wemm "fpupnp", 865398dbb11SPeter Wemm fpupnp_methods, 86654f1d0ceSGarrett Wollman 1, /* no softc */ 86754f1d0ceSGarrett Wollman }; 86854f1d0ceSGarrett Wollman 869398dbb11SPeter Wemm static devclass_t fpupnp_devclass; 87054f1d0ceSGarrett Wollman 871398dbb11SPeter Wemm DRIVER_MODULE(fpupnp, acpi, fpupnp_driver, fpupnp_devclass, 0, 0); 872586079ccSBruce Evans #endif /* DEV_ISA */ 8736cf9a08dSKonstantin Belousov 8748c6f8f3dSKonstantin Belousov static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", 8758c6f8f3dSKonstantin Belousov "Kernel contexts for FPU state"); 8768c6f8f3dSKonstantin Belousov 8778c6f8f3dSKonstantin Belousov #define FPU_KERN_CTX_FPUINITDONE 0x01 8788c6f8f3dSKonstantin Belousov 8798c6f8f3dSKonstantin Belousov struct fpu_kern_ctx { 8808c6f8f3dSKonstantin Belousov struct savefpu *prev; 8818c6f8f3dSKonstantin Belousov uint32_t flags; 8828c6f8f3dSKonstantin Belousov char hwstate1[]; 8838c6f8f3dSKonstantin Belousov }; 8848c6f8f3dSKonstantin Belousov 8858c6f8f3dSKonstantin Belousov struct fpu_kern_ctx * 8868c6f8f3dSKonstantin Belousov fpu_kern_alloc_ctx(u_int flags) 8878c6f8f3dSKonstantin Belousov { 8888c6f8f3dSKonstantin Belousov struct fpu_kern_ctx *res; 8898c6f8f3dSKonstantin Belousov size_t sz; 8908c6f8f3dSKonstantin Belousov 8918c6f8f3dSKonstantin Belousov sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN + 8928c6f8f3dSKonstantin Belousov cpu_max_ext_state_size; 8938c6f8f3dSKonstantin Belousov res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? 8948c6f8f3dSKonstantin Belousov M_NOWAIT : M_WAITOK) | M_ZERO); 8958c6f8f3dSKonstantin Belousov return (res); 8968c6f8f3dSKonstantin Belousov } 8978c6f8f3dSKonstantin Belousov 8988c6f8f3dSKonstantin Belousov void 8998c6f8f3dSKonstantin Belousov fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) 9008c6f8f3dSKonstantin Belousov { 9018c6f8f3dSKonstantin Belousov 9028c6f8f3dSKonstantin Belousov /* XXXKIB clear the memory ? */ 9038c6f8f3dSKonstantin Belousov free(ctx, M_FPUKERN_CTX); 9048c6f8f3dSKonstantin Belousov } 9058c6f8f3dSKonstantin Belousov 9068c6f8f3dSKonstantin Belousov static struct savefpu * 9078c6f8f3dSKonstantin Belousov fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx) 9088c6f8f3dSKonstantin Belousov { 9098c6f8f3dSKonstantin Belousov vm_offset_t p; 9108c6f8f3dSKonstantin Belousov 9118c6f8f3dSKonstantin Belousov p = (vm_offset_t)&ctx->hwstate1; 9128c6f8f3dSKonstantin Belousov p = roundup2(p, XSAVE_AREA_ALIGN); 9138c6f8f3dSKonstantin Belousov return ((struct savefpu *)p); 9148c6f8f3dSKonstantin Belousov } 9158c6f8f3dSKonstantin Belousov 9166cf9a08dSKonstantin Belousov int 9176cf9a08dSKonstantin Belousov fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) 9186cf9a08dSKonstantin Belousov { 9196cf9a08dSKonstantin Belousov struct pcb *pcb; 9206cf9a08dSKonstantin Belousov 9216cf9a08dSKonstantin Belousov pcb = td->td_pcb; 9228c6f8f3dSKonstantin Belousov KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == 9238c6f8f3dSKonstantin Belousov get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); 9246cf9a08dSKonstantin Belousov ctx->flags = 0; 9256cf9a08dSKonstantin Belousov if ((pcb->pcb_flags & PCB_FPUINITDONE) != 0) 9266cf9a08dSKonstantin Belousov ctx->flags |= FPU_KERN_CTX_FPUINITDONE; 9276cf9a08dSKonstantin Belousov fpuexit(td); 9286cf9a08dSKonstantin Belousov ctx->prev = pcb->pcb_save; 9298c6f8f3dSKonstantin Belousov pcb->pcb_save = fpu_kern_ctx_savefpu(ctx); 930e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_KERNFPU); 931e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE); 9326cf9a08dSKonstantin Belousov return (0); 9336cf9a08dSKonstantin Belousov } 9346cf9a08dSKonstantin Belousov 9356cf9a08dSKonstantin Belousov int 9366cf9a08dSKonstantin Belousov fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) 9376cf9a08dSKonstantin Belousov { 9386cf9a08dSKonstantin Belousov struct pcb *pcb; 9396cf9a08dSKonstantin Belousov 9406cf9a08dSKonstantin Belousov pcb = td->td_pcb; 94199753495SKonstantin Belousov critical_enter(); 9426cf9a08dSKonstantin Belousov if (curthread == PCPU_GET(fpcurthread)) 9436cf9a08dSKonstantin Belousov fpudrop(); 94499753495SKonstantin Belousov critical_exit(); 9456cf9a08dSKonstantin Belousov pcb->pcb_save = ctx->prev; 9468c6f8f3dSKonstantin Belousov if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) { 947e6c006d9SJung-uk Kim if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0) { 948e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 949e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_KERNFPU); 950e6c006d9SJung-uk Kim } else 951e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE | PCB_KERNFPU); 9526cf9a08dSKonstantin Belousov } else { 9536cf9a08dSKonstantin Belousov if ((ctx->flags & FPU_KERN_CTX_FPUINITDONE) != 0) 954e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 9556cf9a08dSKonstantin Belousov else 956e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE); 9576cf9a08dSKonstantin Belousov KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); 9586cf9a08dSKonstantin Belousov } 9596cf9a08dSKonstantin Belousov return (0); 9606cf9a08dSKonstantin Belousov } 9616cf9a08dSKonstantin Belousov 9626cf9a08dSKonstantin Belousov int 9636cf9a08dSKonstantin Belousov fpu_kern_thread(u_int flags) 9646cf9a08dSKonstantin Belousov { 9656cf9a08dSKonstantin Belousov 9666cf9a08dSKonstantin Belousov KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, 9676cf9a08dSKonstantin Belousov ("Only kthread may use fpu_kern_thread")); 9681965c139SKonstantin Belousov KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb), 9698c6f8f3dSKonstantin Belousov ("mangled pcb_save")); 9701965c139SKonstantin Belousov KASSERT(PCB_USER_FPU(curpcb), ("recursive call")); 9716cf9a08dSKonstantin Belousov 9721965c139SKonstantin Belousov set_pcb_flags(curpcb, PCB_KERNFPU); 9736cf9a08dSKonstantin Belousov return (0); 9746cf9a08dSKonstantin Belousov } 9756cf9a08dSKonstantin Belousov 9766cf9a08dSKonstantin Belousov int 9776cf9a08dSKonstantin Belousov is_fpu_kern_thread(u_int flags) 9786cf9a08dSKonstantin Belousov { 9796cf9a08dSKonstantin Belousov 9806cf9a08dSKonstantin Belousov if ((curthread->td_pflags & TDP_KTHREAD) == 0) 9816cf9a08dSKonstantin Belousov return (0); 98283b22b05SKonstantin Belousov return ((curpcb->pcb_flags & PCB_KERNFPU) != 0); 9836cf9a08dSKonstantin Belousov } 9842741efecSPeter Grehan 9852741efecSPeter Grehan /* 9862741efecSPeter Grehan * FPU save area alloc/free/init utility routines 9872741efecSPeter Grehan */ 9882741efecSPeter Grehan struct savefpu * 9892741efecSPeter Grehan fpu_save_area_alloc(void) 9902741efecSPeter Grehan { 9912741efecSPeter Grehan 9922741efecSPeter Grehan return (uma_zalloc(fpu_save_area_zone, 0)); 9932741efecSPeter Grehan } 9942741efecSPeter Grehan 9952741efecSPeter Grehan void 9962741efecSPeter Grehan fpu_save_area_free(struct savefpu *fsa) 9972741efecSPeter Grehan { 9982741efecSPeter Grehan 9992741efecSPeter Grehan uma_zfree(fpu_save_area_zone, fsa); 10002741efecSPeter Grehan } 10012741efecSPeter Grehan 10022741efecSPeter Grehan void 10032741efecSPeter Grehan fpu_save_area_reset(struct savefpu *fsa) 10042741efecSPeter Grehan { 10052741efecSPeter Grehan 10062741efecSPeter Grehan bcopy(fpu_initialstate, fsa, cpu_max_ext_state_size); 10072741efecSPeter Grehan } 1008