15b81b6b3SRodney W. Grimes /*- 251369649SPedro F. Giffuni * SPDX-License-Identifier: BSD-3-Clause 351369649SPedro F. Giffuni * 45b81b6b3SRodney W. Grimes * Copyright (c) 1990 William Jolitz. 55b81b6b3SRodney W. Grimes * Copyright (c) 1991 The Regents of the University of California. 65b81b6b3SRodney W. Grimes * All rights reserved. 75b81b6b3SRodney W. Grimes * 85b81b6b3SRodney W. Grimes * Redistribution and use in source and binary forms, with or without 95b81b6b3SRodney W. Grimes * modification, are permitted provided that the following conditions 105b81b6b3SRodney W. Grimes * are met: 115b81b6b3SRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 125b81b6b3SRodney W. Grimes * notice, this list of conditions and the following disclaimer. 135b81b6b3SRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 145b81b6b3SRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 155b81b6b3SRodney W. Grimes * documentation and/or other materials provided with the distribution. 16fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 175b81b6b3SRodney W. Grimes * may be used to endorse or promote products derived from this software 185b81b6b3SRodney W. Grimes * without specific prior written permission. 195b81b6b3SRodney W. Grimes * 205b81b6b3SRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 215b81b6b3SRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 225b81b6b3SRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 235b81b6b3SRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 245b81b6b3SRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 255b81b6b3SRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 265b81b6b3SRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 275b81b6b3SRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 285b81b6b3SRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 295b81b6b3SRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 305b81b6b3SRodney W. Grimes * SUCH DAMAGE. 315b81b6b3SRodney W. Grimes * 3221616ec3SPeter Wemm * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 335b81b6b3SRodney W. Grimes */ 345b81b6b3SRodney W. Grimes 3556ae44c5SDavid E. O'Brien #include <sys/cdefs.h> 3656ae44c5SDavid E. O'Brien __FBSDID("$FreeBSD$"); 3756ae44c5SDavid E. O'Brien 38f540b106SGarrett Wollman #include <sys/param.h> 39f540b106SGarrett Wollman #include <sys/systm.h> 406182fdbdSPeter Wemm #include <sys/bus.h> 413a34a5c3SPoul-Henning Kamp #include <sys/kernel.h> 42fb919e4dSMark Murray #include <sys/lock.h> 43cd59d49dSBruce Evans #include <sys/malloc.h> 446182fdbdSPeter Wemm #include <sys/module.h> 45c1ef8aacSJake Burkholder #include <sys/mutex.h> 46fb919e4dSMark Murray #include <sys/mutex.h> 47fb919e4dSMark Murray #include <sys/proc.h> 48fb919e4dSMark Murray #include <sys/sysctl.h> 496182fdbdSPeter Wemm #include <machine/bus.h> 506182fdbdSPeter Wemm #include <sys/rman.h> 51663f1485SBruce Evans #include <sys/signalvar.h> 522741efecSPeter Grehan #include <vm/uma.h> 532f86936aSGarrett Wollman 547f47cf2fSBruce Evans #include <machine/cputypes.h> 557f47cf2fSBruce Evans #include <machine/frame.h> 560d2a2989SPeter Wemm #include <machine/intr_machdep.h> 57c673fe98SBruce Evans #include <machine/md_var.h> 585400ed3bSPeter Wemm #include <machine/pcb.h> 597f47cf2fSBruce Evans #include <machine/psl.h> 606182fdbdSPeter Wemm #include <machine/resource.h> 61f540b106SGarrett Wollman #include <machine/specialreg.h> 627f47cf2fSBruce Evans #include <machine/segments.h> 6330abe507SJonathan Mini #include <machine/ucontext.h> 642f86936aSGarrett Wollman 655b81b6b3SRodney W. Grimes /* 66bf2f09eeSPeter Wemm * Floating point support. 675b81b6b3SRodney W. Grimes */ 685b81b6b3SRodney W. Grimes 69a5f50ef9SJoerg Wunsch #if defined(__GNUCLIKE_ASM) && !defined(lint) 705b81b6b3SRodney W. Grimes 7117275403SJung-uk Kim #define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) 7230402401SJung-uk Kim #define fnclex() __asm __volatile("fnclex") 7330402401SJung-uk Kim #define fninit() __asm __volatile("fninit") 741d37f051SBruce Evans #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) 752e50fa36SJung-uk Kim #define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) 7630402401SJung-uk Kim #define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) 779d146ac5SPeter Wemm #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) 7807c86dcfSJung-uk Kim #define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) 79a81f9fedSKonstantin Belousov #define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) 805b81b6b3SRodney W. Grimes 8194818d19SKonstantin Belousov static __inline void 8294818d19SKonstantin Belousov xrstor(char *addr, uint64_t mask) 8394818d19SKonstantin Belousov { 8494818d19SKonstantin Belousov uint32_t low, hi; 8594818d19SKonstantin Belousov 8694818d19SKonstantin Belousov low = mask; 8794818d19SKonstantin Belousov hi = mask >> 32; 887574a595SJohn Baldwin __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); 8994818d19SKonstantin Belousov } 9094818d19SKonstantin Belousov 9194818d19SKonstantin Belousov static __inline void 9294818d19SKonstantin Belousov xsave(char *addr, uint64_t mask) 9394818d19SKonstantin Belousov { 9494818d19SKonstantin Belousov uint32_t low, hi; 9594818d19SKonstantin Belousov 9694818d19SKonstantin Belousov low = mask; 9794818d19SKonstantin Belousov hi = mask >> 32; 987574a595SJohn Baldwin __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : 997574a595SJohn Baldwin "memory"); 10094818d19SKonstantin Belousov } 10194818d19SKonstantin Belousov 102cf4e1c46SPeter Wemm #else /* !(__GNUCLIKE_ASM && !lint) */ 1035b81b6b3SRodney W. Grimes 10417275403SJung-uk Kim void fldcw(u_short cw); 10589c9a483SAlfred Perlstein void fnclex(void); 10689c9a483SAlfred Perlstein void fninit(void); 10789c9a483SAlfred Perlstein void fnstcw(caddr_t addr); 10889c9a483SAlfred Perlstein void fnstsw(caddr_t addr); 10989c9a483SAlfred Perlstein void fxsave(caddr_t addr); 11089c9a483SAlfred Perlstein void fxrstor(caddr_t addr); 11107c86dcfSJung-uk Kim void ldmxcsr(u_int csr); 112a42fa0afSKonstantin Belousov void stmxcsr(u_int *csr); 11394818d19SKonstantin Belousov void xrstor(char *addr, uint64_t mask); 11494818d19SKonstantin Belousov void xsave(char *addr, uint64_t mask); 1155b81b6b3SRodney W. Grimes 116cf4e1c46SPeter Wemm #endif /* __GNUCLIKE_ASM && !lint */ 1175b81b6b3SRodney W. Grimes 118d706ec29SJohn Baldwin #define start_emulating() load_cr0(rcr0() | CR0_TS) 119d706ec29SJohn Baldwin #define stop_emulating() clts() 120d706ec29SJohn Baldwin 1218c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct savefpu) == 512); 1228c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct xstate_hdr) == 64); 1238c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct savefpu_ymm) == 832); 1248c6f8f3dSKonstantin Belousov 1258c6f8f3dSKonstantin Belousov /* 1268c6f8f3dSKonstantin Belousov * This requirement is to make it easier for asm code to calculate 1278c6f8f3dSKonstantin Belousov * offset of the fpu save area from the pcb address. FPU save area 128b74a2290SKonstantin Belousov * must be 64-byte aligned. 1298c6f8f3dSKonstantin Belousov */ 1308c6f8f3dSKonstantin Belousov CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0); 1315b81b6b3SRodney W. Grimes 132180e57e5SJohn Baldwin /* 133180e57e5SJohn Baldwin * Ensure the copy of XCR0 saved in a core is contained in the padding 134180e57e5SJohn Baldwin * area. 135180e57e5SJohn Baldwin */ 136180e57e5SJohn Baldwin CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savefpu, sv_pad) && 137180e57e5SJohn Baldwin X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savefpu)); 138180e57e5SJohn Baldwin 1392652af56SColin Percival static void fpu_clean_state(void); 1402652af56SColin Percival 1410b7dc0a7SJohn Baldwin SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, 142f0188618SHans Petter Selasky SYSCTL_NULL_INT_PTR, 1, "Floating point instructions executed in hardware"); 1433a34a5c3SPoul-Henning Kamp 1448c6f8f3dSKonstantin Belousov int use_xsave; /* non-static for cpu_switch.S */ 1458c6f8f3dSKonstantin Belousov uint64_t xsave_mask; /* the same */ 1462741efecSPeter Grehan static uma_zone_t fpu_save_area_zone; 1478c6f8f3dSKonstantin Belousov static struct savefpu *fpu_initialstate; 1488c6f8f3dSKonstantin Belousov 149333d0c60SKonstantin Belousov struct xsave_area_elm_descr { 150333d0c60SKonstantin Belousov u_int offset; 151333d0c60SKonstantin Belousov u_int size; 152333d0c60SKonstantin Belousov } *xsave_area_desc; 153333d0c60SKonstantin Belousov 1548c6f8f3dSKonstantin Belousov void 1558c6f8f3dSKonstantin Belousov fpusave(void *addr) 1568c6f8f3dSKonstantin Belousov { 1578c6f8f3dSKonstantin Belousov 1588c6f8f3dSKonstantin Belousov if (use_xsave) 1598c6f8f3dSKonstantin Belousov xsave((char *)addr, xsave_mask); 1608c6f8f3dSKonstantin Belousov else 1618c6f8f3dSKonstantin Belousov fxsave((char *)addr); 1628c6f8f3dSKonstantin Belousov } 1638c6f8f3dSKonstantin Belousov 1642741efecSPeter Grehan void 1658c6f8f3dSKonstantin Belousov fpurestore(void *addr) 1668c6f8f3dSKonstantin Belousov { 1678c6f8f3dSKonstantin Belousov 1688c6f8f3dSKonstantin Belousov if (use_xsave) 1698c6f8f3dSKonstantin Belousov xrstor((char *)addr, xsave_mask); 1708c6f8f3dSKonstantin Belousov else 1718c6f8f3dSKonstantin Belousov fxrstor((char *)addr); 1728c6f8f3dSKonstantin Belousov } 1733902c3efSSteve Passe 1741d22d877SJung-uk Kim void 1751d22d877SJung-uk Kim fpususpend(void *addr) 1761d22d877SJung-uk Kim { 1771d22d877SJung-uk Kim u_long cr0; 1781d22d877SJung-uk Kim 1791d22d877SJung-uk Kim cr0 = rcr0(); 1801d22d877SJung-uk Kim stop_emulating(); 1811d22d877SJung-uk Kim fpusave(addr); 1821d22d877SJung-uk Kim load_cr0(cr0); 1831d22d877SJung-uk Kim } 1841d22d877SJung-uk Kim 185b1d735baSJohn Baldwin void 186b1d735baSJohn Baldwin fpuresume(void *addr) 187b1d735baSJohn Baldwin { 188b1d735baSJohn Baldwin u_long cr0; 189b1d735baSJohn Baldwin 190b1d735baSJohn Baldwin cr0 = rcr0(); 191b1d735baSJohn Baldwin stop_emulating(); 192b1d735baSJohn Baldwin fninit(); 193b1d735baSJohn Baldwin if (use_xsave) 194b1d735baSJohn Baldwin load_xcr(XCR0, xsave_mask); 195b1d735baSJohn Baldwin fpurestore(addr); 196b1d735baSJohn Baldwin load_cr0(cr0); 197b1d735baSJohn Baldwin } 198b1d735baSJohn Baldwin 1995b81b6b3SRodney W. Grimes /* 2008c6f8f3dSKonstantin Belousov * Enable XSAVE if supported and allowed by user. 2018c6f8f3dSKonstantin Belousov * Calculate the xsave_mask. 2028c6f8f3dSKonstantin Belousov */ 2038c6f8f3dSKonstantin Belousov static void 2048c6f8f3dSKonstantin Belousov fpuinit_bsp1(void) 2058c6f8f3dSKonstantin Belousov { 2068c6f8f3dSKonstantin Belousov u_int cp[4]; 2078c6f8f3dSKonstantin Belousov uint64_t xsave_mask_user; 2089cffc92cSKonstantin Belousov bool old_wp; 2098c6f8f3dSKonstantin Belousov 2108c6f8f3dSKonstantin Belousov if ((cpu_feature2 & CPUID2_XSAVE) != 0) { 2118c6f8f3dSKonstantin Belousov use_xsave = 1; 2128c6f8f3dSKonstantin Belousov TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); 2138c6f8f3dSKonstantin Belousov } 2148c6f8f3dSKonstantin Belousov if (!use_xsave) 2158c6f8f3dSKonstantin Belousov return; 2168c6f8f3dSKonstantin Belousov 2178c6f8f3dSKonstantin Belousov cpuid_count(0xd, 0x0, cp); 2188c6f8f3dSKonstantin Belousov xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; 2198c6f8f3dSKonstantin Belousov if ((cp[0] & xsave_mask) != xsave_mask) 2208c6f8f3dSKonstantin Belousov panic("CPU0 does not support X87 or SSE: %x", cp[0]); 2218c6f8f3dSKonstantin Belousov xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; 2228c6f8f3dSKonstantin Belousov xsave_mask_user = xsave_mask; 2238c6f8f3dSKonstantin Belousov TUNABLE_ULONG_FETCH("hw.xsave_mask", &xsave_mask_user); 2248c6f8f3dSKonstantin Belousov xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; 2258c6f8f3dSKonstantin Belousov xsave_mask &= xsave_mask_user; 2260eb7ae8dSJohn Baldwin if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512) 2270eb7ae8dSJohn Baldwin xsave_mask &= ~XFEATURE_AVX512; 2280eb7ae8dSJohn Baldwin if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX) 2290eb7ae8dSJohn Baldwin xsave_mask &= ~XFEATURE_MPX; 230333d0c60SKonstantin Belousov 231333d0c60SKonstantin Belousov cpuid_count(0xd, 0x1, cp); 232333d0c60SKonstantin Belousov if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) { 233333d0c60SKonstantin Belousov /* 234333d0c60SKonstantin Belousov * Patch the XSAVE instruction in the cpu_switch code 235333d0c60SKonstantin Belousov * to XSAVEOPT. We assume that XSAVE encoding used 236333d0c60SKonstantin Belousov * REX byte, and set the bit 4 of the r/m byte. 2379cffc92cSKonstantin Belousov * 2389cffc92cSKonstantin Belousov * It seems that some BIOSes give control to the OS 2399cffc92cSKonstantin Belousov * with CR0.WP already set, making the kernel text 2409cffc92cSKonstantin Belousov * read-only before cpu_startup(). 241333d0c60SKonstantin Belousov */ 2429cffc92cSKonstantin Belousov old_wp = disable_wp(); 243333d0c60SKonstantin Belousov ctx_switch_xsave[3] |= 0x10; 2449cffc92cSKonstantin Belousov restore_wp(old_wp); 245333d0c60SKonstantin Belousov } 2468c6f8f3dSKonstantin Belousov } 2478c6f8f3dSKonstantin Belousov 2488c6f8f3dSKonstantin Belousov /* 2498c6f8f3dSKonstantin Belousov * Calculate the fpu save area size. 2508c6f8f3dSKonstantin Belousov */ 2518c6f8f3dSKonstantin Belousov static void 2528c6f8f3dSKonstantin Belousov fpuinit_bsp2(void) 2538c6f8f3dSKonstantin Belousov { 2548c6f8f3dSKonstantin Belousov u_int cp[4]; 2558c6f8f3dSKonstantin Belousov 2568c6f8f3dSKonstantin Belousov if (use_xsave) { 2578c6f8f3dSKonstantin Belousov cpuid_count(0xd, 0x0, cp); 2588c6f8f3dSKonstantin Belousov cpu_max_ext_state_size = cp[1]; 2598c6f8f3dSKonstantin Belousov 2608c6f8f3dSKonstantin Belousov /* 2618c6f8f3dSKonstantin Belousov * Reload the cpu_feature2, since we enabled OSXSAVE. 2628c6f8f3dSKonstantin Belousov */ 2638c6f8f3dSKonstantin Belousov do_cpuid(1, cp); 2648c6f8f3dSKonstantin Belousov cpu_feature2 = cp[2]; 2658c6f8f3dSKonstantin Belousov } else 2668c6f8f3dSKonstantin Belousov cpu_max_ext_state_size = sizeof(struct savefpu); 2678c6f8f3dSKonstantin Belousov } 2688c6f8f3dSKonstantin Belousov 2698c6f8f3dSKonstantin Belousov /* 2708c6f8f3dSKonstantin Belousov * Initialize the floating point unit. 271da4113b3SPeter Wemm */ 272398dbb11SPeter Wemm void 2731c89210cSPeter Wemm fpuinit(void) 274da4113b3SPeter Wemm { 2750689bdccSJohn Baldwin register_t saveintr; 27696a7759eSPeter Wemm u_int mxcsr; 277398dbb11SPeter Wemm u_short control; 278da4113b3SPeter Wemm 2798c6f8f3dSKonstantin Belousov if (IS_BSP()) 2808c6f8f3dSKonstantin Belousov fpuinit_bsp1(); 2818c6f8f3dSKonstantin Belousov 2828c6f8f3dSKonstantin Belousov if (use_xsave) { 2838c6f8f3dSKonstantin Belousov load_cr4(rcr4() | CR4_XSAVE); 2847574a595SJohn Baldwin load_xcr(XCR0, xsave_mask); 2858c6f8f3dSKonstantin Belousov } 2868c6f8f3dSKonstantin Belousov 2878c6f8f3dSKonstantin Belousov /* 2888c6f8f3dSKonstantin Belousov * XCR0 shall be set up before CPU can report the save area size. 2898c6f8f3dSKonstantin Belousov */ 2908c6f8f3dSKonstantin Belousov if (IS_BSP()) 2918c6f8f3dSKonstantin Belousov fpuinit_bsp2(); 2928c6f8f3dSKonstantin Belousov 29399753495SKonstantin Belousov /* 29499753495SKonstantin Belousov * It is too early for critical_enter() to work on AP. 29599753495SKonstantin Belousov */ 2960689bdccSJohn Baldwin saveintr = intr_disable(); 2975b81b6b3SRodney W. Grimes stop_emulating(); 2985b81b6b3SRodney W. Grimes fninit(); 299398dbb11SPeter Wemm control = __INITIAL_FPUCW__; 30017275403SJung-uk Kim fldcw(control); 30196a7759eSPeter Wemm mxcsr = __INITIAL_MXCSR__; 30296a7759eSPeter Wemm ldmxcsr(mxcsr); 303a8346a98SJohn Baldwin start_emulating(); 3040689bdccSJohn Baldwin intr_restore(saveintr); 3055b81b6b3SRodney W. Grimes } 3065b81b6b3SRodney W. Grimes 3075b81b6b3SRodney W. Grimes /* 3088c6f8f3dSKonstantin Belousov * On the boot CPU we generate a clean state that is used to 3098c6f8f3dSKonstantin Belousov * initialize the floating point unit when it is first used by a 3108c6f8f3dSKonstantin Belousov * process. 3118c6f8f3dSKonstantin Belousov */ 3128c6f8f3dSKonstantin Belousov static void 3138c6f8f3dSKonstantin Belousov fpuinitstate(void *arg __unused) 3148c6f8f3dSKonstantin Belousov { 3158c6f8f3dSKonstantin Belousov register_t saveintr; 316333d0c60SKonstantin Belousov int cp[4], i, max_ext_n; 3178c6f8f3dSKonstantin Belousov 3188c6f8f3dSKonstantin Belousov fpu_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF, 3198c6f8f3dSKonstantin Belousov M_WAITOK | M_ZERO); 3208c6f8f3dSKonstantin Belousov saveintr = intr_disable(); 3218c6f8f3dSKonstantin Belousov stop_emulating(); 3228c6f8f3dSKonstantin Belousov 3238c6f8f3dSKonstantin Belousov fpusave(fpu_initialstate); 3248c6f8f3dSKonstantin Belousov if (fpu_initialstate->sv_env.en_mxcsr_mask) 3258c6f8f3dSKonstantin Belousov cpu_mxcsr_mask = fpu_initialstate->sv_env.en_mxcsr_mask; 3268c6f8f3dSKonstantin Belousov else 3278c6f8f3dSKonstantin Belousov cpu_mxcsr_mask = 0xFFBF; 3288c6f8f3dSKonstantin Belousov 3298c6f8f3dSKonstantin Belousov /* 330b57e6814SKonstantin Belousov * The fninit instruction does not modify XMM registers or x87 331b57e6814SKonstantin Belousov * registers (MM/ST). The fpusave call dumped the garbage 332b57e6814SKonstantin Belousov * contained in the registers after reset to the initial state 333b57e6814SKonstantin Belousov * saved. Clear XMM and x87 registers file image to make the 334b57e6814SKonstantin Belousov * startup program state and signal handler XMM/x87 register 335b57e6814SKonstantin Belousov * content predictable. 3368c6f8f3dSKonstantin Belousov */ 337b57e6814SKonstantin Belousov bzero(fpu_initialstate->sv_fp, sizeof(fpu_initialstate->sv_fp)); 338b57e6814SKonstantin Belousov bzero(fpu_initialstate->sv_xmm, sizeof(fpu_initialstate->sv_xmm)); 3398c6f8f3dSKonstantin Belousov 340333d0c60SKonstantin Belousov /* 341333d0c60SKonstantin Belousov * Create a table describing the layout of the CPU Extended 342333d0c60SKonstantin Belousov * Save Area. 343333d0c60SKonstantin Belousov */ 34414f52559SKonstantin Belousov if (use_xsave) { 345333d0c60SKonstantin Belousov max_ext_n = flsl(xsave_mask); 346333d0c60SKonstantin Belousov xsave_area_desc = malloc(max_ext_n * sizeof(struct 347333d0c60SKonstantin Belousov xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO); 348333d0c60SKonstantin Belousov /* x87 state */ 349333d0c60SKonstantin Belousov xsave_area_desc[0].offset = 0; 350333d0c60SKonstantin Belousov xsave_area_desc[0].size = 160; 351333d0c60SKonstantin Belousov /* XMM */ 352333d0c60SKonstantin Belousov xsave_area_desc[1].offset = 160; 353333d0c60SKonstantin Belousov xsave_area_desc[1].size = 288 - 160; 354333d0c60SKonstantin Belousov 355333d0c60SKonstantin Belousov for (i = 2; i < max_ext_n; i++) { 356333d0c60SKonstantin Belousov cpuid_count(0xd, i, cp); 357333d0c60SKonstantin Belousov xsave_area_desc[i].offset = cp[1]; 358333d0c60SKonstantin Belousov xsave_area_desc[i].size = cp[0]; 359333d0c60SKonstantin Belousov } 360333d0c60SKonstantin Belousov } 361333d0c60SKonstantin Belousov 3622741efecSPeter Grehan fpu_save_area_zone = uma_zcreate("FPU_save_area", 3632741efecSPeter Grehan cpu_max_ext_state_size, NULL, NULL, NULL, NULL, 3642741efecSPeter Grehan XSAVE_AREA_ALIGN - 1, 0); 3652741efecSPeter Grehan 3668c6f8f3dSKonstantin Belousov start_emulating(); 3678c6f8f3dSKonstantin Belousov intr_restore(saveintr); 3688c6f8f3dSKonstantin Belousov } 369*ad456dd9SKyle Evans /* EFIRT needs this to be initialized before we can enter our EFI environment */ 370*ad456dd9SKyle Evans SYSINIT(fpuinitstate, SI_SUB_DRIVERS, SI_ORDER_FIRST, fpuinitstate, NULL); 3718c6f8f3dSKonstantin Belousov 3728c6f8f3dSKonstantin Belousov /* 3735b81b6b3SRodney W. Grimes * Free coprocessor (if we have it). 3745b81b6b3SRodney W. Grimes */ 3755b81b6b3SRodney W. Grimes void 376bf2f09eeSPeter Wemm fpuexit(struct thread *td) 3775b81b6b3SRodney W. Grimes { 3785b81b6b3SRodney W. Grimes 37999753495SKonstantin Belousov critical_enter(); 3801c89210cSPeter Wemm if (curthread == PCPU_GET(fpcurthread)) { 3811c89210cSPeter Wemm stop_emulating(); 38283b22b05SKonstantin Belousov fpusave(curpcb->pcb_save); 3831c89210cSPeter Wemm start_emulating(); 3846dfc9e44SKonstantin Belousov PCPU_SET(fpcurthread, NULL); 3851c89210cSPeter Wemm } 38699753495SKonstantin Belousov critical_exit(); 3875b81b6b3SRodney W. Grimes } 3885b81b6b3SRodney W. Grimes 38930abe507SJonathan Mini int 390f132cd05SKonstantin Belousov fpuformat(void) 39130abe507SJonathan Mini { 39230abe507SJonathan Mini 39330abe507SJonathan Mini return (_MC_FPFMT_XMM); 39430abe507SJonathan Mini } 39530abe507SJonathan Mini 3965b81b6b3SRodney W. Grimes /* 397a7674320SMartin Cracauer * The following mechanism is used to ensure that the FPE_... value 398a7674320SMartin Cracauer * that is passed as a trapcode to the signal handler of the user 399a7674320SMartin Cracauer * process does not have more than one bit set. 400a7674320SMartin Cracauer * 401a7674320SMartin Cracauer * Multiple bits may be set if the user process modifies the control 402a7674320SMartin Cracauer * word while a status word bit is already set. While this is a sign 403a7674320SMartin Cracauer * of bad coding, we have no choise than to narrow them down to one 404a7674320SMartin Cracauer * bit, since we must not send a trapcode that is not exactly one of 405a7674320SMartin Cracauer * the FPE_ macros. 406a7674320SMartin Cracauer * 407a7674320SMartin Cracauer * The mechanism has a static table with 127 entries. Each combination 408a7674320SMartin Cracauer * of the 7 FPU status word exception bits directly translates to a 409a7674320SMartin Cracauer * position in this table, where a single FPE_... value is stored. 410a7674320SMartin Cracauer * This FPE_... value stored there is considered the "most important" 411a7674320SMartin Cracauer * of the exception bits and will be sent as the signal code. The 412a7674320SMartin Cracauer * precedence of the bits is based upon Intel Document "Numerical 413a7674320SMartin Cracauer * Applications", Chapter "Special Computational Situations". 414a7674320SMartin Cracauer * 415a7674320SMartin Cracauer * The macro to choose one of these values does these steps: 1) Throw 416a7674320SMartin Cracauer * away status word bits that cannot be masked. 2) Throw away the bits 417a7674320SMartin Cracauer * currently masked in the control word, assuming the user isn't 418a7674320SMartin Cracauer * interested in them anymore. 3) Reinsert status word bit 7 (stack 419a7674320SMartin Cracauer * fault) if it is set, which cannot be masked but must be presered. 420a7674320SMartin Cracauer * 4) Use the remaining bits to point into the trapcode table. 421a7674320SMartin Cracauer * 422a7674320SMartin Cracauer * The 6 maskable bits in order of their preference, as stated in the 423a7674320SMartin Cracauer * above referenced Intel manual: 424a7674320SMartin Cracauer * 1 Invalid operation (FP_X_INV) 425a7674320SMartin Cracauer * 1a Stack underflow 426a7674320SMartin Cracauer * 1b Stack overflow 427a7674320SMartin Cracauer * 1c Operand of unsupported format 428a7674320SMartin Cracauer * 1d SNaN operand. 429a7674320SMartin Cracauer * 2 QNaN operand (not an exception, irrelavant here) 430a7674320SMartin Cracauer * 3 Any other invalid-operation not mentioned above or zero divide 431a7674320SMartin Cracauer * (FP_X_INV, FP_X_DZ) 432a7674320SMartin Cracauer * 4 Denormal operand (FP_X_DNML) 433a7674320SMartin Cracauer * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) 434784648c6SMartin Cracauer * 6 Inexact result (FP_X_IMP) 435784648c6SMartin Cracauer */ 436a7674320SMartin Cracauer static char fpetable[128] = { 437a7674320SMartin Cracauer 0, 438a7674320SMartin Cracauer FPE_FLTINV, /* 1 - INV */ 439a7674320SMartin Cracauer FPE_FLTUND, /* 2 - DNML */ 440a7674320SMartin Cracauer FPE_FLTINV, /* 3 - INV | DNML */ 441a7674320SMartin Cracauer FPE_FLTDIV, /* 4 - DZ */ 442a7674320SMartin Cracauer FPE_FLTINV, /* 5 - INV | DZ */ 443a7674320SMartin Cracauer FPE_FLTDIV, /* 6 - DNML | DZ */ 444a7674320SMartin Cracauer FPE_FLTINV, /* 7 - INV | DNML | DZ */ 445a7674320SMartin Cracauer FPE_FLTOVF, /* 8 - OFL */ 446a7674320SMartin Cracauer FPE_FLTINV, /* 9 - INV | OFL */ 447a7674320SMartin Cracauer FPE_FLTUND, /* A - DNML | OFL */ 448a7674320SMartin Cracauer FPE_FLTINV, /* B - INV | DNML | OFL */ 449a7674320SMartin Cracauer FPE_FLTDIV, /* C - DZ | OFL */ 450a7674320SMartin Cracauer FPE_FLTINV, /* D - INV | DZ | OFL */ 451a7674320SMartin Cracauer FPE_FLTDIV, /* E - DNML | DZ | OFL */ 452a7674320SMartin Cracauer FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ 453a7674320SMartin Cracauer FPE_FLTUND, /* 10 - UFL */ 454a7674320SMartin Cracauer FPE_FLTINV, /* 11 - INV | UFL */ 455a7674320SMartin Cracauer FPE_FLTUND, /* 12 - DNML | UFL */ 456a7674320SMartin Cracauer FPE_FLTINV, /* 13 - INV | DNML | UFL */ 457a7674320SMartin Cracauer FPE_FLTDIV, /* 14 - DZ | UFL */ 458a7674320SMartin Cracauer FPE_FLTINV, /* 15 - INV | DZ | UFL */ 459a7674320SMartin Cracauer FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ 460a7674320SMartin Cracauer FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ 461a7674320SMartin Cracauer FPE_FLTOVF, /* 18 - OFL | UFL */ 462a7674320SMartin Cracauer FPE_FLTINV, /* 19 - INV | OFL | UFL */ 463a7674320SMartin Cracauer FPE_FLTUND, /* 1A - DNML | OFL | UFL */ 464a7674320SMartin Cracauer FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ 465a7674320SMartin Cracauer FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ 466a7674320SMartin Cracauer FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ 467a7674320SMartin Cracauer FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ 468a7674320SMartin Cracauer FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ 469a7674320SMartin Cracauer FPE_FLTRES, /* 20 - IMP */ 470a7674320SMartin Cracauer FPE_FLTINV, /* 21 - INV | IMP */ 471a7674320SMartin Cracauer FPE_FLTUND, /* 22 - DNML | IMP */ 472a7674320SMartin Cracauer FPE_FLTINV, /* 23 - INV | DNML | IMP */ 473a7674320SMartin Cracauer FPE_FLTDIV, /* 24 - DZ | IMP */ 474a7674320SMartin Cracauer FPE_FLTINV, /* 25 - INV | DZ | IMP */ 475a7674320SMartin Cracauer FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ 476a7674320SMartin Cracauer FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ 477a7674320SMartin Cracauer FPE_FLTOVF, /* 28 - OFL | IMP */ 478a7674320SMartin Cracauer FPE_FLTINV, /* 29 - INV | OFL | IMP */ 479a7674320SMartin Cracauer FPE_FLTUND, /* 2A - DNML | OFL | IMP */ 480a7674320SMartin Cracauer FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ 481a7674320SMartin Cracauer FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ 482a7674320SMartin Cracauer FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ 483a7674320SMartin Cracauer FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ 484a7674320SMartin Cracauer FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ 485a7674320SMartin Cracauer FPE_FLTUND, /* 30 - UFL | IMP */ 486a7674320SMartin Cracauer FPE_FLTINV, /* 31 - INV | UFL | IMP */ 487a7674320SMartin Cracauer FPE_FLTUND, /* 32 - DNML | UFL | IMP */ 488a7674320SMartin Cracauer FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ 489a7674320SMartin Cracauer FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ 490a7674320SMartin Cracauer FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ 491a7674320SMartin Cracauer FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ 492a7674320SMartin Cracauer FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ 493a7674320SMartin Cracauer FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ 494a7674320SMartin Cracauer FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ 495a7674320SMartin Cracauer FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ 496a7674320SMartin Cracauer FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ 497a7674320SMartin Cracauer FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ 498a7674320SMartin Cracauer FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ 499a7674320SMartin Cracauer FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ 500a7674320SMartin Cracauer FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ 501a7674320SMartin Cracauer FPE_FLTSUB, /* 40 - STK */ 502a7674320SMartin Cracauer FPE_FLTSUB, /* 41 - INV | STK */ 503a7674320SMartin Cracauer FPE_FLTUND, /* 42 - DNML | STK */ 504a7674320SMartin Cracauer FPE_FLTSUB, /* 43 - INV | DNML | STK */ 505a7674320SMartin Cracauer FPE_FLTDIV, /* 44 - DZ | STK */ 506a7674320SMartin Cracauer FPE_FLTSUB, /* 45 - INV | DZ | STK */ 507a7674320SMartin Cracauer FPE_FLTDIV, /* 46 - DNML | DZ | STK */ 508a7674320SMartin Cracauer FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ 509a7674320SMartin Cracauer FPE_FLTOVF, /* 48 - OFL | STK */ 510a7674320SMartin Cracauer FPE_FLTSUB, /* 49 - INV | OFL | STK */ 511a7674320SMartin Cracauer FPE_FLTUND, /* 4A - DNML | OFL | STK */ 512a7674320SMartin Cracauer FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ 513a7674320SMartin Cracauer FPE_FLTDIV, /* 4C - DZ | OFL | STK */ 514a7674320SMartin Cracauer FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ 515a7674320SMartin Cracauer FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ 516a7674320SMartin Cracauer FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ 517a7674320SMartin Cracauer FPE_FLTUND, /* 50 - UFL | STK */ 518a7674320SMartin Cracauer FPE_FLTSUB, /* 51 - INV | UFL | STK */ 519a7674320SMartin Cracauer FPE_FLTUND, /* 52 - DNML | UFL | STK */ 520a7674320SMartin Cracauer FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ 521a7674320SMartin Cracauer FPE_FLTDIV, /* 54 - DZ | UFL | STK */ 522a7674320SMartin Cracauer FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ 523a7674320SMartin Cracauer FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ 524a7674320SMartin Cracauer FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ 525a7674320SMartin Cracauer FPE_FLTOVF, /* 58 - OFL | UFL | STK */ 526a7674320SMartin Cracauer FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ 527a7674320SMartin Cracauer FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ 528a7674320SMartin Cracauer FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ 529a7674320SMartin Cracauer FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ 530a7674320SMartin Cracauer FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ 531a7674320SMartin Cracauer FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ 532a7674320SMartin Cracauer FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ 533a7674320SMartin Cracauer FPE_FLTRES, /* 60 - IMP | STK */ 534a7674320SMartin Cracauer FPE_FLTSUB, /* 61 - INV | IMP | STK */ 535a7674320SMartin Cracauer FPE_FLTUND, /* 62 - DNML | IMP | STK */ 536a7674320SMartin Cracauer FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ 537a7674320SMartin Cracauer FPE_FLTDIV, /* 64 - DZ | IMP | STK */ 538a7674320SMartin Cracauer FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ 539a7674320SMartin Cracauer FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ 540a7674320SMartin Cracauer FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ 541a7674320SMartin Cracauer FPE_FLTOVF, /* 68 - OFL | IMP | STK */ 542a7674320SMartin Cracauer FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ 543a7674320SMartin Cracauer FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ 544a7674320SMartin Cracauer FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ 545a7674320SMartin Cracauer FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ 546a7674320SMartin Cracauer FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ 547a7674320SMartin Cracauer FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ 548a7674320SMartin Cracauer FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ 549a7674320SMartin Cracauer FPE_FLTUND, /* 70 - UFL | IMP | STK */ 550a7674320SMartin Cracauer FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ 551a7674320SMartin Cracauer FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ 552a7674320SMartin Cracauer FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ 553a7674320SMartin Cracauer FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ 554a7674320SMartin Cracauer FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ 555a7674320SMartin Cracauer FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ 556a7674320SMartin Cracauer FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ 557a7674320SMartin Cracauer FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ 558a7674320SMartin Cracauer FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ 559a7674320SMartin Cracauer FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ 560a7674320SMartin Cracauer FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ 561a7674320SMartin Cracauer FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ 562a7674320SMartin Cracauer FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ 563a7674320SMartin Cracauer FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ 564a7674320SMartin Cracauer FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ 565a7674320SMartin Cracauer }; 566a7674320SMartin Cracauer 567a7674320SMartin Cracauer /* 568dfa8a512SKonstantin Belousov * Read the FP status and control words, then generate si_code value 569dfa8a512SKonstantin Belousov * for SIGFPE. The error code chosen will be one of the 570dfa8a512SKonstantin Belousov * FPE_... macros. It will be sent as the second argument to old 571dfa8a512SKonstantin Belousov * BSD-style signal handlers and as "siginfo_t->si_code" (second 572dfa8a512SKonstantin Belousov * argument) to SA_SIGINFO signal handlers. 5735b81b6b3SRodney W. Grimes * 574dfa8a512SKonstantin Belousov * Some time ago, we cleared the x87 exceptions with FNCLEX there. 575dfa8a512SKonstantin Belousov * Clearing exceptions was necessary mainly to avoid IRQ13 bugs. The 576dfa8a512SKonstantin Belousov * usermode code which understands the FPU hardware enough to enable 577dfa8a512SKonstantin Belousov * the exceptions, can also handle clearing the exception state in the 578dfa8a512SKonstantin Belousov * handler. The only consequence of not clearing the exception is the 579dfa8a512SKonstantin Belousov * rethrow of the SIGFPE on return from the signal handler and 580dfa8a512SKonstantin Belousov * reexecution of the corresponding instruction. 581bc84db62SKonstantin Belousov * 582dfa8a512SKonstantin Belousov * For XMM traps, the exceptions were never cleared. 5835b81b6b3SRodney W. Grimes */ 5841c1771cbSBruce Evans int 585bc84db62SKonstantin Belousov fputrap_x87(void) 5865b81b6b3SRodney W. Grimes { 587bc84db62SKonstantin Belousov struct savefpu *pcb_save; 5881c1771cbSBruce Evans u_short control, status; 5895b81b6b3SRodney W. Grimes 59099753495SKonstantin Belousov critical_enter(); 5915b81b6b3SRodney W. Grimes 5925b81b6b3SRodney W. Grimes /* 5931c1771cbSBruce Evans * Interrupt handling (for another interrupt) may have pushed the 5941c1771cbSBruce Evans * state to memory. Fetch the relevant parts of the state from 5951c1771cbSBruce Evans * wherever they are. 5965b81b6b3SRodney W. Grimes */ 5970bbc8826SJohn Baldwin if (PCPU_GET(fpcurthread) != curthread) { 59883b22b05SKonstantin Belousov pcb_save = curpcb->pcb_save; 599bc84db62SKonstantin Belousov control = pcb_save->sv_env.en_cw; 600bc84db62SKonstantin Belousov status = pcb_save->sv_env.en_sw; 6015b81b6b3SRodney W. Grimes } else { 6021c1771cbSBruce Evans fnstcw(&control); 6031c1771cbSBruce Evans fnstsw(&status); 6045b81b6b3SRodney W. Grimes } 6051c1771cbSBruce Evans 60699753495SKonstantin Belousov critical_exit(); 6071c1771cbSBruce Evans return (fpetable[status & ((~control & 0x3f) | 0x40)]); 6085b81b6b3SRodney W. Grimes } 6095b81b6b3SRodney W. Grimes 610bc84db62SKonstantin Belousov int 611bc84db62SKonstantin Belousov fputrap_sse(void) 612bc84db62SKonstantin Belousov { 613bc84db62SKonstantin Belousov u_int mxcsr; 614bc84db62SKonstantin Belousov 615bc84db62SKonstantin Belousov critical_enter(); 616bc84db62SKonstantin Belousov if (PCPU_GET(fpcurthread) != curthread) 61783b22b05SKonstantin Belousov mxcsr = curpcb->pcb_save->sv_env.en_mxcsr; 618bc84db62SKonstantin Belousov else 619bc84db62SKonstantin Belousov stmxcsr(&mxcsr); 620bc84db62SKonstantin Belousov critical_exit(); 621bc84db62SKonstantin Belousov return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]); 622bc84db62SKonstantin Belousov } 623bc84db62SKonstantin Belousov 6246dfc9e44SKonstantin Belousov /* 6256dfc9e44SKonstantin Belousov * Device Not Available (DNA, #NM) exception handler. 6266dfc9e44SKonstantin Belousov * 6276dfc9e44SKonstantin Belousov * It would be better to switch FP context here (if curthread != 6286dfc9e44SKonstantin Belousov * fpcurthread) and not necessarily for every context switch, but it 6296dfc9e44SKonstantin Belousov * is too hard to access foreign pcb's. 6306dfc9e44SKonstantin Belousov */ 631a8346a98SJohn Baldwin void 632a8346a98SJohn Baldwin fpudna(void) 6335b81b6b3SRodney W. Grimes { 63405f6ee66SJake Burkholder 635060cd4d5SKonstantin Belousov /* 636060cd4d5SKonstantin Belousov * This handler is entered with interrupts enabled, so context 637060cd4d5SKonstantin Belousov * switches may occur before critical_enter() is executed. If 638060cd4d5SKonstantin Belousov * a context switch occurs, then when we regain control, our 639060cd4d5SKonstantin Belousov * state will have been completely restored. The CPU may 640060cd4d5SKonstantin Belousov * change underneath us, but the only part of our context that 641060cd4d5SKonstantin Belousov * lives in the CPU is CR0.TS and that will be "restored" by 642060cd4d5SKonstantin Belousov * setting it on the new CPU. 643060cd4d5SKonstantin Belousov */ 64499753495SKonstantin Belousov critical_enter(); 645060cd4d5SKonstantin Belousov 646cf1c4776SKonstantin Belousov KASSERT((curpcb->pcb_flags & PCB_FPUNOSAVE) == 0, 647cf1c4776SKonstantin Belousov ("fpudna while in fpu_kern_enter(FPU_KERN_NOCTX)")); 64830abe507SJonathan Mini if (PCPU_GET(fpcurthread) == curthread) { 649060cd4d5SKonstantin Belousov printf("fpudna: fpcurthread == curthread\n"); 65030abe507SJonathan Mini stop_emulating(); 65199753495SKonstantin Belousov critical_exit(); 652a8346a98SJohn Baldwin return; 65330abe507SJonathan Mini } 6540bbc8826SJohn Baldwin if (PCPU_GET(fpcurthread) != NULL) { 6556dfc9e44SKonstantin Belousov panic("fpudna: fpcurthread = %p (%d), curthread = %p (%d)\n", 6566dfc9e44SKonstantin Belousov PCPU_GET(fpcurthread), PCPU_GET(fpcurthread)->td_tid, 6576dfc9e44SKonstantin Belousov curthread, curthread->td_tid); 6585b81b6b3SRodney W. Grimes } 6595b81b6b3SRodney W. Grimes stop_emulating(); 6605b81b6b3SRodney W. Grimes /* 661bf2f09eeSPeter Wemm * Record new context early in case frstor causes a trap. 6625b81b6b3SRodney W. Grimes */ 6630bbc8826SJohn Baldwin PCPU_SET(fpcurthread, curthread); 6649d146ac5SPeter Wemm 6652652af56SColin Percival fpu_clean_state(); 6662652af56SColin Percival 6671965c139SKonstantin Belousov if ((curpcb->pcb_flags & PCB_FPUINITDONE) == 0) { 6685b81b6b3SRodney W. Grimes /* 66963de9515SJohn Baldwin * This is the first time this thread has used the FPU or 67063de9515SJohn Baldwin * the PCB doesn't contain a clean FPU state. Explicitly 67163de9515SJohn Baldwin * load an initial state. 672333d0c60SKonstantin Belousov * 673333d0c60SKonstantin Belousov * We prefer to restore the state from the actual save 674333d0c60SKonstantin Belousov * area in PCB instead of directly loading from 675333d0c60SKonstantin Belousov * fpu_initialstate, to ignite the XSAVEOPT 676333d0c60SKonstantin Belousov * tracking engine. 6775b81b6b3SRodney W. Grimes */ 678f132cd05SKonstantin Belousov bcopy(fpu_initialstate, curpcb->pcb_save, 679f132cd05SKonstantin Belousov cpu_max_ext_state_size); 6801965c139SKonstantin Belousov fpurestore(curpcb->pcb_save); 6811965c139SKonstantin Belousov if (curpcb->pcb_initial_fpucw != __INITIAL_FPUCW__) 6821965c139SKonstantin Belousov fldcw(curpcb->pcb_initial_fpucw); 6831965c139SKonstantin Belousov if (PCB_USER_FPU(curpcb)) 6841965c139SKonstantin Belousov set_pcb_flags(curpcb, 685e6c006d9SJung-uk Kim PCB_FPUINITDONE | PCB_USERFPUINITDONE); 686e6c006d9SJung-uk Kim else 6871965c139SKonstantin Belousov set_pcb_flags(curpcb, PCB_FPUINITDONE); 6881c89210cSPeter Wemm } else 6891965c139SKonstantin Belousov fpurestore(curpcb->pcb_save); 69099753495SKonstantin Belousov critical_exit(); 6915b81b6b3SRodney W. Grimes } 6925b81b6b3SRodney W. Grimes 69330abe507SJonathan Mini void 694f132cd05SKonstantin Belousov fpudrop(void) 69530abe507SJonathan Mini { 69630abe507SJonathan Mini struct thread *td; 69730abe507SJonathan Mini 69830abe507SJonathan Mini td = PCPU_GET(fpcurthread); 69999753495SKonstantin Belousov KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); 7004a23ecc7SKonstantin Belousov CRITICAL_ASSERT(td); 70130abe507SJonathan Mini PCPU_SET(fpcurthread, NULL); 702e6c006d9SJung-uk Kim clear_pcb_flags(td->td_pcb, PCB_FPUINITDONE); 70330abe507SJonathan Mini start_emulating(); 70430abe507SJonathan Mini } 70530abe507SJonathan Mini 70630abe507SJonathan Mini /* 7075c6eb037SKonstantin Belousov * Get the user state of the FPU into pcb->pcb_user_save without 7085c6eb037SKonstantin Belousov * dropping ownership (if possible). It returns the FPU ownership 7095c6eb037SKonstantin Belousov * status. 71030abe507SJonathan Mini */ 71130abe507SJonathan Mini int 7125c6eb037SKonstantin Belousov fpugetregs(struct thread *td) 7136cf9a08dSKonstantin Belousov { 7146cf9a08dSKonstantin Belousov struct pcb *pcb; 715333d0c60SKonstantin Belousov uint64_t *xstate_bv, bit; 716333d0c60SKonstantin Belousov char *sa; 71714f52559SKonstantin Belousov int max_ext_n, i, owned; 7186cf9a08dSKonstantin Belousov 7196cf9a08dSKonstantin Belousov pcb = td->td_pcb; 7206cf9a08dSKonstantin Belousov if ((pcb->pcb_flags & PCB_USERFPUINITDONE) == 0) { 7218c6f8f3dSKonstantin Belousov bcopy(fpu_initialstate, get_pcb_user_save_pcb(pcb), 7228c6f8f3dSKonstantin Belousov cpu_max_ext_state_size); 7238c6f8f3dSKonstantin Belousov get_pcb_user_save_pcb(pcb)->sv_env.en_cw = 7248c6f8f3dSKonstantin Belousov pcb->pcb_initial_fpucw; 7255c6eb037SKonstantin Belousov fpuuserinited(td); 7265c6eb037SKonstantin Belousov return (_MC_FPOWNED_PCB); 7276cf9a08dSKonstantin Belousov } 72899753495SKonstantin Belousov critical_enter(); 7296cf9a08dSKonstantin Belousov if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 7308c6f8f3dSKonstantin Belousov fpusave(get_pcb_user_save_pcb(pcb)); 73114f52559SKonstantin Belousov owned = _MC_FPOWNED_FPU; 7326cf9a08dSKonstantin Belousov } else { 73314f52559SKonstantin Belousov owned = _MC_FPOWNED_PCB; 73414f52559SKonstantin Belousov } 73599753495SKonstantin Belousov critical_exit(); 73614f52559SKonstantin Belousov if (use_xsave) { 737333d0c60SKonstantin Belousov /* 738333d0c60SKonstantin Belousov * Handle partially saved state. 739333d0c60SKonstantin Belousov */ 740333d0c60SKonstantin Belousov sa = (char *)get_pcb_user_save_pcb(pcb); 741333d0c60SKonstantin Belousov xstate_bv = (uint64_t *)(sa + sizeof(struct savefpu) + 742333d0c60SKonstantin Belousov offsetof(struct xstate_hdr, xstate_bv)); 743333d0c60SKonstantin Belousov max_ext_n = flsl(xsave_mask); 744333d0c60SKonstantin Belousov for (i = 0; i < max_ext_n; i++) { 745241b67bbSKonstantin Belousov bit = 1ULL << i; 746241b67bbSKonstantin Belousov if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0) 747333d0c60SKonstantin Belousov continue; 748333d0c60SKonstantin Belousov bcopy((char *)fpu_initialstate + 749333d0c60SKonstantin Belousov xsave_area_desc[i].offset, 750333d0c60SKonstantin Belousov sa + xsave_area_desc[i].offset, 751333d0c60SKonstantin Belousov xsave_area_desc[i].size); 752333d0c60SKonstantin Belousov *xstate_bv |= bit; 753333d0c60SKonstantin Belousov } 754333d0c60SKonstantin Belousov } 75514f52559SKonstantin Belousov return (owned); 7566cf9a08dSKonstantin Belousov } 7576cf9a08dSKonstantin Belousov 7585c6eb037SKonstantin Belousov void 7595c6eb037SKonstantin Belousov fpuuserinited(struct thread *td) 76030abe507SJonathan Mini { 7616cf9a08dSKonstantin Belousov struct pcb *pcb; 76230abe507SJonathan Mini 7636cf9a08dSKonstantin Belousov pcb = td->td_pcb; 7645c6eb037SKonstantin Belousov if (PCB_USER_FPU(pcb)) 765e6c006d9SJung-uk Kim set_pcb_flags(pcb, 766e6c006d9SJung-uk Kim PCB_FPUINITDONE | PCB_USERFPUINITDONE); 767e6c006d9SJung-uk Kim else 768e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 76930abe507SJonathan Mini } 77030abe507SJonathan Mini 7718c6f8f3dSKonstantin Belousov int 7728c6f8f3dSKonstantin Belousov fpusetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size) 7738c6f8f3dSKonstantin Belousov { 7748c6f8f3dSKonstantin Belousov struct xstate_hdr *hdr, *ehdr; 7758c6f8f3dSKonstantin Belousov size_t len, max_len; 7768c6f8f3dSKonstantin Belousov uint64_t bv; 7778c6f8f3dSKonstantin Belousov 7788c6f8f3dSKonstantin Belousov /* XXXKIB should we clear all extended state in xstate_bv instead ? */ 7798c6f8f3dSKonstantin Belousov if (xfpustate == NULL) 7808c6f8f3dSKonstantin Belousov return (0); 7818c6f8f3dSKonstantin Belousov if (!use_xsave) 7828c6f8f3dSKonstantin Belousov return (EOPNOTSUPP); 7838c6f8f3dSKonstantin Belousov 7848c6f8f3dSKonstantin Belousov len = xfpustate_size; 7858c6f8f3dSKonstantin Belousov if (len < sizeof(struct xstate_hdr)) 7868c6f8f3dSKonstantin Belousov return (EINVAL); 7878c6f8f3dSKonstantin Belousov max_len = cpu_max_ext_state_size - sizeof(struct savefpu); 7888c6f8f3dSKonstantin Belousov if (len > max_len) 7898c6f8f3dSKonstantin Belousov return (EINVAL); 7908c6f8f3dSKonstantin Belousov 7918c6f8f3dSKonstantin Belousov ehdr = (struct xstate_hdr *)xfpustate; 7928c6f8f3dSKonstantin Belousov bv = ehdr->xstate_bv; 7938c6f8f3dSKonstantin Belousov 7948c6f8f3dSKonstantin Belousov /* 7958c6f8f3dSKonstantin Belousov * Avoid #gp. 7968c6f8f3dSKonstantin Belousov */ 7978c6f8f3dSKonstantin Belousov if (bv & ~xsave_mask) 7988c6f8f3dSKonstantin Belousov return (EINVAL); 7998c6f8f3dSKonstantin Belousov 8008c6f8f3dSKonstantin Belousov hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1); 8018c6f8f3dSKonstantin Belousov 8028c6f8f3dSKonstantin Belousov hdr->xstate_bv = bv; 8038c6f8f3dSKonstantin Belousov bcopy(xfpustate + sizeof(struct xstate_hdr), 8048c6f8f3dSKonstantin Belousov (char *)(hdr + 1), len - sizeof(struct xstate_hdr)); 8058c6f8f3dSKonstantin Belousov 8068c6f8f3dSKonstantin Belousov return (0); 8078c6f8f3dSKonstantin Belousov } 8088c6f8f3dSKonstantin Belousov 80930abe507SJonathan Mini /* 81030abe507SJonathan Mini * Set the state of the FPU. 81130abe507SJonathan Mini */ 8128c6f8f3dSKonstantin Belousov int 8138c6f8f3dSKonstantin Belousov fpusetregs(struct thread *td, struct savefpu *addr, char *xfpustate, 8148c6f8f3dSKonstantin Belousov size_t xfpustate_size) 8156cf9a08dSKonstantin Belousov { 8166cf9a08dSKonstantin Belousov struct pcb *pcb; 8178c6f8f3dSKonstantin Belousov int error; 8186cf9a08dSKonstantin Belousov 819aa788cc3SKonstantin Belousov addr->sv_env.en_mxcsr &= cpu_mxcsr_mask; 8206cf9a08dSKonstantin Belousov pcb = td->td_pcb; 82199753495SKonstantin Belousov critical_enter(); 8226cf9a08dSKonstantin Belousov if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 8238c6f8f3dSKonstantin Belousov error = fpusetxstate(td, xfpustate, xfpustate_size); 8248c6f8f3dSKonstantin Belousov if (error != 0) { 8258c6f8f3dSKonstantin Belousov critical_exit(); 8268c6f8f3dSKonstantin Belousov return (error); 8278c6f8f3dSKonstantin Belousov } 8288c6f8f3dSKonstantin Belousov bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); 8298c6f8f3dSKonstantin Belousov fpurestore(get_pcb_user_save_td(td)); 83099753495SKonstantin Belousov critical_exit(); 831e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE | PCB_USERFPUINITDONE); 8326cf9a08dSKonstantin Belousov } else { 83399753495SKonstantin Belousov critical_exit(); 8348c6f8f3dSKonstantin Belousov error = fpusetxstate(td, xfpustate, xfpustate_size); 8358c6f8f3dSKonstantin Belousov if (error != 0) 8368c6f8f3dSKonstantin Belousov return (error); 8378c6f8f3dSKonstantin Belousov bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); 8385c6eb037SKonstantin Belousov fpuuserinited(td); 8396cf9a08dSKonstantin Belousov } 8408c6f8f3dSKonstantin Belousov return (0); 8416cf9a08dSKonstantin Belousov } 8426cf9a08dSKonstantin Belousov 8436182fdbdSPeter Wemm /* 8442652af56SColin Percival * On AuthenticAMD processors, the fxrstor instruction does not restore 8452652af56SColin Percival * the x87's stored last instruction pointer, last data pointer, and last 8462652af56SColin Percival * opcode values, except in the rare case in which the exception summary 8472652af56SColin Percival * (ES) bit in the x87 status word is set to 1. 8482652af56SColin Percival * 8492652af56SColin Percival * In order to avoid leaking this information across processes, we clean 8502652af56SColin Percival * these values by performing a dummy load before executing fxrstor(). 8512652af56SColin Percival */ 8522652af56SColin Percival static void 8532652af56SColin Percival fpu_clean_state(void) 8542652af56SColin Percival { 855b9dda9d6SJohn Baldwin static float dummy_variable = 0.0; 8562652af56SColin Percival u_short status; 8572652af56SColin Percival 8582652af56SColin Percival /* 8592652af56SColin Percival * Clear the ES bit in the x87 status word if it is currently 8602652af56SColin Percival * set, in order to avoid causing a fault in the upcoming load. 8612652af56SColin Percival */ 8622652af56SColin Percival fnstsw(&status); 8632652af56SColin Percival if (status & 0x80) 8642652af56SColin Percival fnclex(); 8652652af56SColin Percival 8662652af56SColin Percival /* 8672652af56SColin Percival * Load the dummy variable into the x87 stack. This mangles 8682652af56SColin Percival * the x87 stack, but we don't care since we're about to call 8692652af56SColin Percival * fxrstor() anyway. 8702652af56SColin Percival */ 87114965052SDimitry Andric __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable)); 8722652af56SColin Percival } 8732652af56SColin Percival 8742652af56SColin Percival /* 875398dbb11SPeter Wemm * This really sucks. We want the acpi version only, but it requires 876398dbb11SPeter Wemm * the isa_if.h file in order to get the definitions. 8776182fdbdSPeter Wemm */ 878398dbb11SPeter Wemm #include "opt_isa.h" 879afa88623SPeter Wemm #ifdef DEV_ISA 880398dbb11SPeter Wemm #include <isa/isavar.h> 88154f1d0ceSGarrett Wollman /* 8825f063c7bSMike Smith * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI. 88354f1d0ceSGarrett Wollman */ 884398dbb11SPeter Wemm static struct isa_pnp_id fpupnp_ids[] = { 88554f1d0ceSGarrett Wollman { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */ 88654f1d0ceSGarrett Wollman { 0 } 88754f1d0ceSGarrett Wollman }; 88854f1d0ceSGarrett Wollman 88954f1d0ceSGarrett Wollman static int 890398dbb11SPeter Wemm fpupnp_probe(device_t dev) 89154f1d0ceSGarrett Wollman { 892bb9c06c1SMike Smith int result; 893bf2f09eeSPeter Wemm 894398dbb11SPeter Wemm result = ISA_PNP_PROBE(device_get_parent(dev), dev, fpupnp_ids); 895bf2f09eeSPeter Wemm if (result <= 0) 896bb9c06c1SMike Smith device_quiet(dev); 897bb9c06c1SMike Smith return (result); 89854f1d0ceSGarrett Wollman } 89954f1d0ceSGarrett Wollman 90054f1d0ceSGarrett Wollman static int 901398dbb11SPeter Wemm fpupnp_attach(device_t dev) 90254f1d0ceSGarrett Wollman { 903bf2f09eeSPeter Wemm 90454f1d0ceSGarrett Wollman return (0); 90554f1d0ceSGarrett Wollman } 90654f1d0ceSGarrett Wollman 907398dbb11SPeter Wemm static device_method_t fpupnp_methods[] = { 90854f1d0ceSGarrett Wollman /* Device interface */ 909398dbb11SPeter Wemm DEVMETHOD(device_probe, fpupnp_probe), 910398dbb11SPeter Wemm DEVMETHOD(device_attach, fpupnp_attach), 91154f1d0ceSGarrett Wollman DEVMETHOD(device_detach, bus_generic_detach), 91254f1d0ceSGarrett Wollman DEVMETHOD(device_shutdown, bus_generic_shutdown), 91354f1d0ceSGarrett Wollman DEVMETHOD(device_suspend, bus_generic_suspend), 91454f1d0ceSGarrett Wollman DEVMETHOD(device_resume, bus_generic_resume), 91554f1d0ceSGarrett Wollman 91654f1d0ceSGarrett Wollman { 0, 0 } 91754f1d0ceSGarrett Wollman }; 91854f1d0ceSGarrett Wollman 919398dbb11SPeter Wemm static driver_t fpupnp_driver = { 920398dbb11SPeter Wemm "fpupnp", 921398dbb11SPeter Wemm fpupnp_methods, 92254f1d0ceSGarrett Wollman 1, /* no softc */ 92354f1d0ceSGarrett Wollman }; 92454f1d0ceSGarrett Wollman 925398dbb11SPeter Wemm static devclass_t fpupnp_devclass; 92654f1d0ceSGarrett Wollman 927398dbb11SPeter Wemm DRIVER_MODULE(fpupnp, acpi, fpupnp_driver, fpupnp_devclass, 0, 0); 928d6b66397SWarner Losh ISA_PNP_INFO(fpupnp_ids); 929586079ccSBruce Evans #endif /* DEV_ISA */ 9306cf9a08dSKonstantin Belousov 9318c6f8f3dSKonstantin Belousov static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", 9328c6f8f3dSKonstantin Belousov "Kernel contexts for FPU state"); 9338c6f8f3dSKonstantin Belousov 9348c6f8f3dSKonstantin Belousov #define FPU_KERN_CTX_FPUINITDONE 0x01 935633034feSKonstantin Belousov #define FPU_KERN_CTX_DUMMY 0x02 /* avoided save for the kern thread */ 936e808e13bSJohn-Mark Gurney #define FPU_KERN_CTX_INUSE 0x04 9378c6f8f3dSKonstantin Belousov 9388c6f8f3dSKonstantin Belousov struct fpu_kern_ctx { 9398c6f8f3dSKonstantin Belousov struct savefpu *prev; 9408c6f8f3dSKonstantin Belousov uint32_t flags; 9418c6f8f3dSKonstantin Belousov char hwstate1[]; 9428c6f8f3dSKonstantin Belousov }; 9438c6f8f3dSKonstantin Belousov 9448c6f8f3dSKonstantin Belousov struct fpu_kern_ctx * 9458c6f8f3dSKonstantin Belousov fpu_kern_alloc_ctx(u_int flags) 9468c6f8f3dSKonstantin Belousov { 9478c6f8f3dSKonstantin Belousov struct fpu_kern_ctx *res; 9488c6f8f3dSKonstantin Belousov size_t sz; 9498c6f8f3dSKonstantin Belousov 9508c6f8f3dSKonstantin Belousov sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN + 9518c6f8f3dSKonstantin Belousov cpu_max_ext_state_size; 9528c6f8f3dSKonstantin Belousov res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? 9538c6f8f3dSKonstantin Belousov M_NOWAIT : M_WAITOK) | M_ZERO); 9548c6f8f3dSKonstantin Belousov return (res); 9558c6f8f3dSKonstantin Belousov } 9568c6f8f3dSKonstantin Belousov 9578c6f8f3dSKonstantin Belousov void 9588c6f8f3dSKonstantin Belousov fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) 9598c6f8f3dSKonstantin Belousov { 9608c6f8f3dSKonstantin Belousov 961e808e13bSJohn-Mark Gurney KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx")); 9628c6f8f3dSKonstantin Belousov /* XXXKIB clear the memory ? */ 9638c6f8f3dSKonstantin Belousov free(ctx, M_FPUKERN_CTX); 9648c6f8f3dSKonstantin Belousov } 9658c6f8f3dSKonstantin Belousov 9668c6f8f3dSKonstantin Belousov static struct savefpu * 9678c6f8f3dSKonstantin Belousov fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx) 9688c6f8f3dSKonstantin Belousov { 9698c6f8f3dSKonstantin Belousov vm_offset_t p; 9708c6f8f3dSKonstantin Belousov 9718c6f8f3dSKonstantin Belousov p = (vm_offset_t)&ctx->hwstate1; 9728c6f8f3dSKonstantin Belousov p = roundup2(p, XSAVE_AREA_ALIGN); 9738c6f8f3dSKonstantin Belousov return ((struct savefpu *)p); 9748c6f8f3dSKonstantin Belousov } 9758c6f8f3dSKonstantin Belousov 976849ce31aSConrad Meyer void 9776cf9a08dSKonstantin Belousov fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) 9786cf9a08dSKonstantin Belousov { 9796cf9a08dSKonstantin Belousov struct pcb *pcb; 9806cf9a08dSKonstantin Belousov 981cf1c4776SKonstantin Belousov pcb = td->td_pcb; 982cf1c4776SKonstantin Belousov KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL, 983cf1c4776SKonstantin Belousov ("ctx is required when !FPU_KERN_NOCTX")); 984cf1c4776SKonstantin Belousov KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, 985cf1c4776SKonstantin Belousov ("using inuse ctx")); 986cf1c4776SKonstantin Belousov KASSERT((pcb->pcb_flags & PCB_FPUNOSAVE) == 0, 987cf1c4776SKonstantin Belousov ("recursive fpu_kern_enter while in PCB_FPUNOSAVE state")); 988e808e13bSJohn-Mark Gurney 989cf1c4776SKonstantin Belousov if ((flags & FPU_KERN_NOCTX) != 0) { 990cf1c4776SKonstantin Belousov critical_enter(); 991cf1c4776SKonstantin Belousov stop_emulating(); 992cf1c4776SKonstantin Belousov if (curthread == PCPU_GET(fpcurthread)) { 993cf1c4776SKonstantin Belousov fpusave(curpcb->pcb_save); 994cf1c4776SKonstantin Belousov PCPU_SET(fpcurthread, NULL); 995cf1c4776SKonstantin Belousov } else { 996cf1c4776SKonstantin Belousov KASSERT(PCPU_GET(fpcurthread) == NULL, 997cf1c4776SKonstantin Belousov ("invalid fpcurthread")); 998cf1c4776SKonstantin Belousov } 999cf1c4776SKonstantin Belousov 1000cf1c4776SKonstantin Belousov /* 1001cf1c4776SKonstantin Belousov * This breaks XSAVEOPT tracker, but 1002cf1c4776SKonstantin Belousov * PCB_FPUNOSAVE state is supposed to never need to 1003cf1c4776SKonstantin Belousov * save FPU context at all. 1004cf1c4776SKonstantin Belousov */ 1005cf1c4776SKonstantin Belousov fpurestore(fpu_initialstate); 1006cf1c4776SKonstantin Belousov set_pcb_flags(pcb, PCB_KERNFPU | PCB_FPUNOSAVE | 1007cf1c4776SKonstantin Belousov PCB_FPUINITDONE); 1008849ce31aSConrad Meyer return; 1009cf1c4776SKonstantin Belousov } 1010633034feSKonstantin Belousov if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { 1011e808e13bSJohn-Mark Gurney ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; 1012849ce31aSConrad Meyer return; 1013633034feSKonstantin Belousov } 10148c6f8f3dSKonstantin Belousov KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == 10158c6f8f3dSKonstantin Belousov get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); 1016e808e13bSJohn-Mark Gurney ctx->flags = FPU_KERN_CTX_INUSE; 10176cf9a08dSKonstantin Belousov if ((pcb->pcb_flags & PCB_FPUINITDONE) != 0) 10186cf9a08dSKonstantin Belousov ctx->flags |= FPU_KERN_CTX_FPUINITDONE; 10196cf9a08dSKonstantin Belousov fpuexit(td); 10206cf9a08dSKonstantin Belousov ctx->prev = pcb->pcb_save; 10218c6f8f3dSKonstantin Belousov pcb->pcb_save = fpu_kern_ctx_savefpu(ctx); 1022e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_KERNFPU); 1023e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE); 1024849ce31aSConrad Meyer return; 10256cf9a08dSKonstantin Belousov } 10266cf9a08dSKonstantin Belousov 10276cf9a08dSKonstantin Belousov int 10286cf9a08dSKonstantin Belousov fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) 10296cf9a08dSKonstantin Belousov { 10306cf9a08dSKonstantin Belousov struct pcb *pcb; 10316cf9a08dSKonstantin Belousov 1032cf1c4776SKonstantin Belousov pcb = td->td_pcb; 1033cf1c4776SKonstantin Belousov 1034cf1c4776SKonstantin Belousov if ((pcb->pcb_flags & PCB_FPUNOSAVE) != 0) { 1035cf1c4776SKonstantin Belousov KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX")); 1036cf1c4776SKonstantin Belousov KASSERT(PCPU_GET(fpcurthread) == NULL, 1037cf1c4776SKonstantin Belousov ("non-NULL fpcurthread for PCB_FPUNOSAVE")); 1038cf1c4776SKonstantin Belousov CRITICAL_ASSERT(td); 1039cf1c4776SKonstantin Belousov 1040cf1c4776SKonstantin Belousov clear_pcb_flags(pcb, PCB_FPUNOSAVE | PCB_FPUINITDONE); 1041cf1c4776SKonstantin Belousov start_emulating(); 1042cf1c4776SKonstantin Belousov critical_exit(); 1043cf1c4776SKonstantin Belousov } else { 1044e808e13bSJohn-Mark Gurney KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, 1045e808e13bSJohn-Mark Gurney ("leaving not inuse ctx")); 1046e808e13bSJohn-Mark Gurney ctx->flags &= ~FPU_KERN_CTX_INUSE; 1047e808e13bSJohn-Mark Gurney 1048cf1c4776SKonstantin Belousov if (is_fpu_kern_thread(0) && 1049cf1c4776SKonstantin Belousov (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) 1050633034feSKonstantin Belousov return (0); 1051cf1c4776SKonstantin Belousov KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, 1052cf1c4776SKonstantin Belousov ("dummy ctx")); 105399753495SKonstantin Belousov critical_enter(); 10546cf9a08dSKonstantin Belousov if (curthread == PCPU_GET(fpcurthread)) 10556cf9a08dSKonstantin Belousov fpudrop(); 105699753495SKonstantin Belousov critical_exit(); 10576cf9a08dSKonstantin Belousov pcb->pcb_save = ctx->prev; 1058cf1c4776SKonstantin Belousov } 1059cf1c4776SKonstantin Belousov 10608c6f8f3dSKonstantin Belousov if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) { 1061e6c006d9SJung-uk Kim if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0) { 1062e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 1063e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_KERNFPU); 1064e6c006d9SJung-uk Kim } else 1065e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE | PCB_KERNFPU); 10666cf9a08dSKonstantin Belousov } else { 10676cf9a08dSKonstantin Belousov if ((ctx->flags & FPU_KERN_CTX_FPUINITDONE) != 0) 1068e6c006d9SJung-uk Kim set_pcb_flags(pcb, PCB_FPUINITDONE); 10696cf9a08dSKonstantin Belousov else 1070e6c006d9SJung-uk Kim clear_pcb_flags(pcb, PCB_FPUINITDONE); 10716cf9a08dSKonstantin Belousov KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); 10726cf9a08dSKonstantin Belousov } 10736cf9a08dSKonstantin Belousov return (0); 10746cf9a08dSKonstantin Belousov } 10756cf9a08dSKonstantin Belousov 10766cf9a08dSKonstantin Belousov int 10776cf9a08dSKonstantin Belousov fpu_kern_thread(u_int flags) 10786cf9a08dSKonstantin Belousov { 10796cf9a08dSKonstantin Belousov 10806cf9a08dSKonstantin Belousov KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, 10816cf9a08dSKonstantin Belousov ("Only kthread may use fpu_kern_thread")); 10821965c139SKonstantin Belousov KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb), 10838c6f8f3dSKonstantin Belousov ("mangled pcb_save")); 10841965c139SKonstantin Belousov KASSERT(PCB_USER_FPU(curpcb), ("recursive call")); 10856cf9a08dSKonstantin Belousov 10861965c139SKonstantin Belousov set_pcb_flags(curpcb, PCB_KERNFPU); 10876cf9a08dSKonstantin Belousov return (0); 10886cf9a08dSKonstantin Belousov } 10896cf9a08dSKonstantin Belousov 10906cf9a08dSKonstantin Belousov int 10916cf9a08dSKonstantin Belousov is_fpu_kern_thread(u_int flags) 10926cf9a08dSKonstantin Belousov { 10936cf9a08dSKonstantin Belousov 10946cf9a08dSKonstantin Belousov if ((curthread->td_pflags & TDP_KTHREAD) == 0) 10956cf9a08dSKonstantin Belousov return (0); 109683b22b05SKonstantin Belousov return ((curpcb->pcb_flags & PCB_KERNFPU) != 0); 10976cf9a08dSKonstantin Belousov } 10982741efecSPeter Grehan 10992741efecSPeter Grehan /* 11002741efecSPeter Grehan * FPU save area alloc/free/init utility routines 11012741efecSPeter Grehan */ 11022741efecSPeter Grehan struct savefpu * 11032741efecSPeter Grehan fpu_save_area_alloc(void) 11042741efecSPeter Grehan { 11052741efecSPeter Grehan 11062741efecSPeter Grehan return (uma_zalloc(fpu_save_area_zone, 0)); 11072741efecSPeter Grehan } 11082741efecSPeter Grehan 11092741efecSPeter Grehan void 11102741efecSPeter Grehan fpu_save_area_free(struct savefpu *fsa) 11112741efecSPeter Grehan { 11122741efecSPeter Grehan 11132741efecSPeter Grehan uma_zfree(fpu_save_area_zone, fsa); 11142741efecSPeter Grehan } 11152741efecSPeter Grehan 11162741efecSPeter Grehan void 11172741efecSPeter Grehan fpu_save_area_reset(struct savefpu *fsa) 11182741efecSPeter Grehan { 11192741efecSPeter Grehan 11202741efecSPeter Grehan bcopy(fpu_initialstate, fsa, cpu_max_ext_state_size); 11212741efecSPeter Grehan } 1122