xref: /illumos-gate/usr/src/uts/intel/os/fpu.c (revision ac2250cb76bb32944fd2c8a3ba2cd3f79747748d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2021 Joyent, Inc.
 * Copyright 2021 RackTop Systems, Inc.
 * Copyright 2026 Oxide Computer Company
 * Copyright 2025 Edgecast Cloud LLC.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T   */
/*		All Rights Reserved				*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation		*/
/*		All Rights Reserved				*/

/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/signal.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/trap.h>
#include <sys/fault.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/pcb.h>
#include <sys/lwp.h>
#include <sys/cpuvar.h>
#include <sys/thread.h>
#include <sys/disp.h>
#include <sys/fp.h>
#include <sys/siginfo.h>
#include <sys/archsystm.h>
#include <sys/kmem.h>
#include <sys/debug.h>
#include <sys/x86_archext.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/kfpu.h>
#include <sys/stdbool.h>
#include <sys/stdalign.h>
#include <sys/procfs_isa.h>
#include <sys/sunddi.h>

/*
 * FPU Management Overview
 * -----------------------
 *
 * The x86 FPU has evolved substantially since its days as the x87 coprocessor;
 * however, many aspects of its life as a coprocessor are still around in x86.
 *
 * Today, when we refer to the 'FPU', we don't just mean the original x87 FPU.
 * While that state still exists, there is much more that is covered by the FPU.
 * Today, this includes not just traditional FPU state, but also supervisor only
 * state. The following state is currently managed and covered logically by the
 * idea of the FPU registers and more generally is called the Extended Processor
 * States:
 *
 *    o Traditional x87 FPU
 *    o Vector Registers (%xmm, %ymm, %zmm)
 *    o Memory Protection Extensions (MPX) Bounds Registers
 *    o Protected Key Rights Registers (PKRU)
 *    o Processor Trace data
 *    o Control-Flow Enforcement state
 *    o Hardware Duty Cycle
 *    o Hardware P-states
 *
 * The rest of this covers how the FPU is managed and controlled, how state is
 * saved and restored between threads, interactions with hypervisors, and other
 * information exported to userland through aux vectors. A lot of background
 * information is here to synthesize major parts of the Intel SDM, but
 * unfortunately, it is not a replacement for reading it.
 *
 * FPU Control Registers
 * ---------------------
 *
 * Because the x87 FPU began its life as a co-processor and the FPU was
 * optional there are several bits that show up in %cr0 that we have to
 * manipulate when dealing with the FPU. These are:
 *
 *   o CR0.ET	The 'extension type' bit. This was used originally to indicate
 *		that the FPU co-processor was present. Now it is forced on for
 *		compatibility. This is often used to verify whether or not the
 *		FPU is present.
 *
 *   o CR0.NE	The 'native error' bit. Used to indicate that native error
 *		mode should be enabled. This indicates that we should take traps
 *		on FPU errors. The OS enables this early in boot.
 *
 *   o CR0.MP	The 'Monitor Coprocessor' bit. Used to control whether or not
 *		wait/fwait instructions generate a #NM if CR0.TS is set.
 *
 *   o CR0.EM	The 'Emulation' bit. This is used to cause floating point
 *		operations (x87 through SSE4) to trap with a #UD so they can be
 *		emulated. The system never sets this bit, but makes sure it is
 *		clear on processor start up.
 *
 *   o CR0.TS	The 'Task Switched' bit. When this is turned on, a floating
 *		point operation will generate a #NM. An fwait will as well,
 *		depending on the value in CR0.MP.
 *
 * Our general policy is that CR0.ET, CR0.NE, and CR0.MP are always set by
 * the system. Similarly CR0.EM is always unset by the system. CR0.TS has a more
 * complicated role. Historically it has been used to allow running systems to
 * restore the FPU registers lazily. This will be discussed in greater depth
 * later on.
 *
 * %cr4 is also used as part of the FPU control. Specifically we need to worry
 * about the following bits in the system:
 *
 *   o CR4.OSFXSR	This bit is used to indicate that the OS understands and
 *			supports the execution of the fxsave and fxrstor
 *			instructions. This bit is required to be set to enable
 *			the use of the SSE->SSE4 instructions.
 *
 *   o CR4.OSXMMEXCPT	This bit is used to indicate that the OS can understand
 *			and take a SIMD floating point exception (#XM). This bit
 *			is always enabled by the system.
 *
 *   o CR4.OSXSAVE	This bit is used to indicate that the OS understands and
 *			supports the execution of the xsave and xrstor family of
 *			instructions. This bit is required to use any of the AVX
 *			and newer feature sets.
 *
 * Because all supported processors are 64-bit, they'll always support the XMM
 * extensions and we will enable both CR4.OXFXSR and CR4.OSXMMEXCPT in boot.
 * CR4.OSXSAVE will be enabled and used whenever xsave is reported in cpuid.
 *
 * %xcr0 is used to manage the behavior of the xsave feature set and is only
 * present on the system if xsave is supported. %xcr0 is read and written to
 * through by the xgetbv and xsetbv instructions. This register is present
 * whenever the xsave feature set is supported. Each bit in %xcr0 refers to a
 * different component of the xsave state and controls whether or not that
 * information is saved and restored. For newer feature sets like AVX and MPX,
 * it also controls whether or not the corresponding instructions can be
 * executed (much like CR0.OSFXSR does for the SSE feature sets).
 *
 * Everything in %xcr0 is around features available to users. There is also the
 * IA32_XSS MSR which is used to control supervisor-only features that are still
 * part of the xsave state. Bits that can be set in %xcr0 are reserved in
 * IA32_XSS and vice versa. This is an important property that is particularly
 * relevant to how the xsave instructions operate.
 *
 * Save Mechanisms
 * ---------------
 *
 * When switching between running threads the FPU state needs to be saved and
 * restored by the OS. If this state was not saved, users would rightfully
 * complain about corrupt state. There are three mechanisms that exist on the
 * processor for saving and restoring these state images:
 *
 *   o fsave
 *   o fxsave
 *   o xsave
 *
 * fsave saves and restores only the x87 FPU and is the oldest of these
 * mechanisms. This mechanism is never used in the kernel today because we are
 * always running on systems that support fxsave.
 *
 * The fxsave and fxrstor mechanism allows the x87 FPU and the SSE register
 * state to be saved and restored to and from a struct fxsave_state. This is the
 * default mechanism that is used to save and restore the FPU on amd64. An
 * important aspect of fxsave that was different from the original i386 fsave
 * mechanism is that the restoring of FPU state with pending exceptions will not
 * generate an exception, it will be deferred to the next use of the FPU.
 *
 * The final and by far the most complex mechanism is that of the xsave set.
 * xsave allows for saving and restoring all of the traditional x86 pieces (x87
 * and SSE), while allowing for extensions that will save the %ymm, %zmm, etc.
 * registers.
 *
 * Data is saved and restored into and out of a struct xsave_state. The first
 * part of the struct xsave_state is equivalent to the struct fxsave_state.
 * After that, there is a header which is used to describe the remaining
 * portions of the state. The header is a 64-byte value of which the first two
 * uint64_t values are defined and the rest are reserved and must be zero. The
 * first uint64_t is the xstate_bv member. This describes which values in the
 * xsave_state are actually valid and present. This is updated on a save and
 * used on restore. The second member is the xcomp_bv member. Its last bit
 * determines whether or not a compressed version of the structure is used.
 *
 * When the uncompressed structure is used (currently the only format we
 * support), then each state component is at a fixed offset in the structure,
 * even if it is not being used. For example, if you only saved the AVX related
 * state, but did not save the MPX related state, the offset would not change
 * for any component. With the compressed format, components that aren't used
 * are all elided (though the x87 and SSE state are always there).
 *
 * Unlike fxsave which saves all state, the xsave family does not always save
 * and restore all the state that could be covered by the xsave_state. The
 * instructions all take an argument which is a mask of what to consider. This
 * is the same mask that will be used in the xstate_bv vector and it is also the
 * same values that are present in %xcr0 and IA32_XSS. Though IA32_XSS is only
 * considered with the xsaves and xrstors instructions.
 *
 * When a save or restore is requested, a bitwise and is performed between the
 * requested bits and those that have been enabled in %xcr0. Only the bits that
 * match that are then saved or restored. Others will be silently ignored by
 * the processor. This idea is used often in the OS. We will always request that
 * we save and restore all of the state, but only those portions that are
 * actually enabled in %xcr0 will be touched.
 *
 * If a feature has been asked to be restored that is not set in the xstate_bv
 * feature vector of the save state, then it will be set to its initial state by
 * the processor (usually zeros). Also, when asked to save state, the processor
 * may not write out data that is in its initial state as an optimization. This
 * optimization only applies to saving data and not to restoring data.
 *
 * There are a few different variants of the xsave and xrstor instruction. They
 * are:
 *
 *   o xsave	This is the original save instruction. It will save all of the
 *		requested data in the xsave state structure. It only saves data
 *		in the uncompressed (xcomp_bv[63] is zero) format. It may be
 *		executed at all privilege levels.
 *
 *   o xrstor	This is the original restore instruction. It will restore all of
 *		the requested data. The xrstor function can handle both the
 *		compressed and uncompressed formats. It may be executed at all
 *		privilege levels.
 *
 *   o xsaveopt	This is a variant of the xsave instruction that employs
 *		optimizations to try and only write out state that has been
 *		modified since the last time an xrstor instruction was called.
 *		The processor tracks a tuple of information about the last
 *		xrstor and tries to ensure that the same buffer is being used
 *		when this optimization is being used. However, because of the
 *		way that it tracks the xrstor buffer based on the address of it,
 *		it is not suitable for use if that buffer can be easily reused.
 *		The most common case is trying to save data to the stack in
 *		rtld. It may be executed at all privilege levels.
 *
 *   o xsavec	This is a variant of the xsave instruction that writes out the
 *		compressed form of the xsave_state. Otherwise it behaves as
 *		xsave. It may be executed at all privilege levels.
 *
 *   o xsaves	This is a variant of the xsave instruction. It is similar to
 *		xsavec in that it always writes the compressed form of the
 *		buffer. Unlike all the other forms, this instruction looks at
 *		both the user (%xcr0) and supervisor (IA32_XSS MSR) to determine
 *		what to save and restore. xsaves also implements the same
 *		optimization that xsaveopt does around modified pieces. User
 *		land may not execute the instruction.
 *
 *   o xrstors	This is a variant of the xrstor instruction. Similar to xsaves
 *		it can save and restore both the user and privileged states.
 *		Unlike xrstor it can only operate on the compressed form.
 *		User land may not execute the instruction.
 *
 * Based on all of these, the kernel has a precedence for what it will use.
 * Basically, xsaves (not supported) is preferred to xsaveopt, which is
 * preferred to xsave. A similar scheme is used when informing rtld (more later)
 * about what it should use. xsavec is preferred to xsave. xsaveopt is not
 * recommended due to the modified optimization not being appropriate for this
 * use.
 *
 * Finally, there is one last gotcha with the xsave state. Importantly some AMD
 * processors did not always save and restore some of the FPU exception state in
 * some cases like Intel did. In those cases the OS will make up for this fact
 * itself.
 *
 * FPU Initialization
 * ------------------
 *
 * One difference with the FPU registers is that not all threads have FPU state,
 * only those that have an lwp. Generally this means kernel threads, which all
 * share p0 and its lwp, do not have FPU state. Though there are definitely
 * exceptions such as kcfpoold. In the rest of this discussion we'll use thread
 * and lwp interchangeably, just think of thread meaning a thread that has a
 * lwp.
 *
 * Each lwp has its FPU state allocated in its pcb (process control block). The
 * actual storage comes from the fpsave_cachep kmem cache. This cache is sized
 * dynamically at start up based on the save mechanism that we're using and the
 * amount of memory required for it. This is dynamic because the xsave_state
 * size varies based on the supported feature set.
 *
 * The hardware side of the FPU is initialized early in boot before we mount the
 * root file system. This is effectively done in fpu_probe(). This is where we
 * make the final decision about what the save and restore mechanisms we should
 * use are, create the fpsave_cachep kmem cache, and initialize a number of
 * function pointers that use save and restoring logic.
 *
 * The thread/lwp side is a a little more involved. There are two different
 * things that we need to concern ourselves with. The first is how the FPU
 * resources are allocated and the second is how the FPU state is initialized
 * for a given lwp.
 *
 * We allocate the FPU save state from our kmem cache as part of lwp_fp_init().
 * This is always called unconditionally by the system as part of creating an
 * LWP.
 *
 * There are three different initialization paths that we deal with. The first
 * is when we are executing a new process. As part of exec all of the register
 * state is reset. The exec case is particularly important because init is born
 * like Athena, sprouting from the head of the kernel, without any true parent
 * to fork from. The second is used whenever we fork or create a new lwp.  The
 * third is to deal with special lwps like the agent lwp.
 *
 * During exec, we will call fp_exec() which will initialize and set up the FPU
 * state for the process. That will fill in the initial state for the FPU and
 * also set that state in the FPU itself. As part of fp_exec() we also install a
 * thread context operations vector that takes care of dealing with the saving
 * and restoring of the FPU. These context handlers will also be called whenever
 * an lwp is created or forked. In those cases, to initialize the FPU we will
 * call fp_new_lwp(). Like fp_exec(), fp_new_lwp() will install a context
 * operations vector for the new thread.
 *
 * Next we'll end up in the context operation fp_new_lwp(). This saves the
 * current thread's state, initializes the new thread's state, and copies over
 * the relevant parts of the originating thread's state. It's as this point that
 * we also install the FPU context operations into the new thread, which ensures
 * that all future threads that are descendants of the current one get the
 * thread context operations (unless they call exec).
 *
 * The spawn(2) system call behind posix_spawn(3C) leans on this exec path too.
 * A spawned child is created as a bare kernel thread that does not go through
 * fork or fp_new_lwp(), so it inherits no FPU register state and no context
 * operations from the spawning thread. Like init, it has no parent to fork from
 * as far as the FPU is concerned. It runs in the kernel until it execs its
 * target, at which point fp_exec() installs the context operations and the
 * initial state. There is nothing to inherit; any state copied from the
 * spawning thread would only be discarded by the exec that always follows.
 *
 * To deal with some things like the agent lwp, we double check the state of the
 * FPU in sys_rtt_common() to make sure that it has been enabled before
 * returning to userland. In general, this path should be rare, but it's useful
 * for the odd lwp here and there.
 *
 * The FPU state will remain valid most of the time. There are times that
 * the state will be rewritten. For example in restorecontext, due to /proc, or
 * the lwp calls exec(). Whether the context is being freed or we are resetting
 * the state, we will call fp_free() to disable the FPU and our context.
 *
 * Finally, when the lwp is destroyed, it will actually destroy and free the FPU
 * state by calling fp_lwp_cleanup().
 *
 * Kernel FPU Multiplexing
 * -----------------------
 *
 * Just as the kernel has to maintain all of the general purpose registers when
 * switching between scheduled threads, the same is true of the FPU registers.
 *
 * When a thread has FPU state, it also has a set of context operations
 * installed. These context operations take care of making sure that the FPU is
 * properly saved and restored during a context switch (fpsave_ctxt and
 * fprestore_ctxt respectively). This means that the current implementation of
 * the FPU is 'eager', when a thread is running the CPU will have its FPU state
 * loaded. While this is always true when executing in userland, there are a few
 * cases where this is not true in the kernel.
 *
 * This was not always the case. Traditionally on x86 a 'lazy' FPU restore was
 * employed. This meant that the FPU would be saved on a context switch and the
 * CR0.TS bit would be set. When a thread next tried to use the FPU, it would
 * then take a #NM trap, at which point we would restore the FPU from the save
 * area and return to userland. Given the frequency of use of the FPU alone by
 * libc, there's no point returning to userland just to trap again.
 *
 * There are a few cases though where the FPU state may need to be changed for a
 * thread on its behalf. The most notable cases are in the case of processes
 * using /proc, restorecontext, forking, etc. In all of these cases the kernel
 * will force a threads FPU state to be saved into the PCB through the fp_save()
 * function. Whenever the FPU is saved, then the FPU_VALID flag is set on the
 * pcb. This indicates that the save state holds currently valid data. As a side
 * effect of this, CR0.TS will be set. To make sure that all of the state is
 * updated before returning to userland, in these cases, we set a flag on the
 * PCB that says the FPU needs to be updated. This will make sure that we take
 * the slow path out of a system call to fix things up for the thread. Due to
 * the fact that this is a rather rare case, effectively setting the equivalent
 * of t_postsys is acceptable.
 *
 * CR0.TS will be set after a save occurs and cleared when a restore occurs.
 * Generally this means it will be cleared immediately by the new thread that is
 * running in a context switch. However, this isn't the case for kernel threads.
 * They currently operate with CR0.TS set as no kernel state is restored for
 * them. This means that using the FPU will cause a #NM and panic.
 *
 * The FPU_VALID flag on the currently executing thread's pcb is meant to track
 * what the value of CR0.TS should be. If it is set, then CR0.TS will be set.
 * However, because we eagerly restore, the only time that CR0.TS should be set
 * for a non-kernel thread is during operations where it will be cleared before
 * returning to userland and importantly, the only data that is in it is its
 * own.
 *
 * Kernel FPU Usage
 * ----------------
 *
 * Traditionally the kernel never used the FPU since it had no need for
 * floating point operations. However, modern FPU hardware supports a variety
 * of SIMD extensions which can speed up code such as parity calculations or
 * encryption.
 *
 * To allow the kernel to take advantage of these features, the
 * kernel_fpu_begin() and kernel_fpu_end() functions should be wrapped
 * around any usage of the FPU by the kernel to ensure that user-level context
 * is properly saved/restored, as well as to properly setup the FPU for use by
 * the kernel. There are a variety of ways this wrapping can be used, as
 * discussed in this section below.
 *
 * When kernel_fpu_begin() and kernel_fpu_end() are used for extended
 * operations, the kernel_fpu_alloc() function should be used to allocate a
 * kfpu_state_t structure that is used to save/restore the thread's kernel FPU
 * state. This structure is not tied to any thread. That is, different threads
 * can reuse the same kfpu_state_t structure, although not concurrently. A
 * kfpu_state_t structure is freed by the kernel_fpu_free() function.
 *
 * In some cases, the kernel may need to use the FPU for a short operation
 * without the overhead to manage a kfpu_state_t structure and without
 * allowing for a context switch off the FPU. In this case the KFPU_NO_STATE
 * bit can be set in the kernel_fpu_begin() and kernel_fpu_end() flags
 * parameter. This indicates that there is no kfpu_state_t. When used this way,
 * kernel preemption should be disabled by the caller (kpreempt_disable) before
 * calling kernel_fpu_begin(), and re-enabled after calling kernel_fpu_end().
 * For this usage, it is important to limit the kernel's FPU use to short
 * operations. The tradeoff between using the FPU without a kfpu_state_t
 * structure vs. the overhead of allowing a context switch while using the FPU
 * should be carefully considered on a case by case basis.
 *
 * In other cases, kernel threads have an LWP, but never execute in user space.
 * In this situation, the LWP's pcb_fpu area can be used to save/restore the
 * kernel's FPU state if the thread is context switched, instead of having to
 * allocate and manage a kfpu_state_t structure. The KFPU_USE_LWP bit in the
 * kernel_fpu_begin() and kernel_fpu_end() flags parameter is used to
 * enable this behavior. It is the caller's responsibility to ensure that this
 * is only used for a kernel thread which never executes in user space.
 *
 * FPU Exceptions
 * --------------
 *
 * Certain operations can cause the kernel to take traps due to FPU activity.
 * Generally these events will cause a user process to receive a SIGFPU and if
 * the kernel receives it in kernel context, we will die. Traditionally the #NM
 * (Device Not Available / No Math) exception generated by CR0.TS would have
 * caused us to restore the FPU. Now it is a fatal event regardless of whether
 * or not userland causes it.
 *
 * While there are some cases where the kernel uses the FPU, it is up to the
 * kernel to use the FPU in a way such that it cannot receive a trap or to use
 * the appropriate trap protection mechanisms.
 *
 * Hypervisors
 * -----------
 *
 * When providing support for hypervisors things are a little bit more
 * complicated because the FPU is not virtualized at all. This means that they
 * need to save and restore the FPU and %xcr0 across entry and exit to the
 * guest. To facilitate this, we provide a series of APIs in <sys/hma.h>. These
 * allow us to use the full native state to make sure that we are always saving
 * and restoring the full FPU that the host sees, even when the guest is using a
 * subset.
 *
 * One tricky aspect of this is that the guest may be using a subset of %xcr0
 * and therefore changing our %xcr0 on the fly. It is vital that when we're
 * saving and restoring the FPU that we always use the largest %xcr0 contents
 * otherwise we will end up leaving behind data in it.
 *
 * ELF PLT Support
 * ---------------
 *
 * rtld has to preserve a subset of the FPU when it is saving and restoring
 * registers due to the amd64 SYS V ABI. See cmd/sgs/rtld/amd64/boot_elf.s for
 * more information. As a result, we set up an aux vector that contains
 * information about what save and restore mechanisms it should be using and
 * the sizing thereof based on what the kernel supports. This is passed down in
 * a series of aux vectors SUN_AT_FPTYPE and SUN_AT_FPSIZE. This information is
 * initialized in fpu_subr.c.
 *
 * Signal Handling and the ucontext_t
 * ----------------------------------
 *
 * One of the many gifts that signals give us is the twofold fact that when a
 * signal occurs, the signal handler is allowed to change the CPU's state
 * arbitrarily and when the signal handler is done executing, we must restore it
 * back to the original state. However, the second part of this is that the
 * signal handler is actually allowed to modify the state that the thread will
 * return to! To create this facade, the kernel will create a full ucontext_t
 * state, effectively calling getcontext(2) on the thread's behalf, and a
 * pointer to that is given to the signal handler (the void * argument for the
 * sa_sigaction function pointer in sigaction(2)). When libc is done with a
 * signal, it will call setcontext(2) with that same ucontext_t.
 *
 * Now, the ucontext_t has a fixed ABI for both ILP32 and LP64 environments and
 * it's often declared on the stack itself, with the signal handler spilling all
 * this state to the stack. The ucontext_t machine portion was broken into the
 * general purpose and floating point registers. In 64-bit code, the floating
 * point registers were mostly the same as the results of the fxsave instruction
 * (i.e. struct fxsave_state). While the 64-bit kernel still uses the equivalent
 * starting point for information, it is transformed into a different shape to
 * deal with the history of the 32-bit SYS V ABI.
 *
 * While this worked, if you're reading this, you're aware that the x86 FPU and
 * extended register states didn't stop at the initial 16 128-bit %xmm
 * registers. Since then we have added 256-bit %ymm, 512-bit %zmm, and the %k
 * opmask registers. None of these fit inside the standard ucontext_t; however,
 * they must all be preserved and restored across a signal. While the various
 * x86 platform-specific ABIs all suggest that these registers are not preserved
 * across a function call, receiving a signal is not a function call and must be
 * thought of like a process receiving an interrupt. In other words, this
 * extended state must be preserved.
 *
 * To facilitate this, we have extended the ucontext_t structure with an
 * additional flag, UC_XSAVE, which indicates that the traditional padding
 * member, uc_xsave, actually is a pointer to the extended state. While this is
 * accessible outside of a signal handling context through the combination of
 * ucontext_alloc(3C) and getcontext_extd(2), our design around saving this
 * state is focused on signal handling. Signal handling spills all this state to
 * the stack and if we cannot spill the entire state to the stack then our
 * inability to deliver the signal results in the process being killed! While
 * there are separate efforts to ensure that the signal stack sizing that is
 * used for the minimum and maximum signal sizes are sufficient, we still need
 * to do our part to minimize the likelihood here.
 *
 * In designing this, we make the following observations which have helped us
 * focus our design:
 *
 *   o While the start of an xsave area is the traditional 512-byte fxsave XMM
 *     region, we already have that in the fpregs. Thus there is no reason to
 *     duplicate it. This not only saves 512 bytes of additional stack space,
 *     but it also means we don't have to ask which of the version of it to take
 *     if they were to differ.
 *
 *   o Many applications out there aren't necessarily using the extended vectors
 *     and even when we do make libc and others take advantage of it, it will
 *     behoove us to ensure that they are put back into their initial state
 *     after use. This leads us to expect that in a number of cases, the actual
 *     extended register state will be in its initial state.
 *
 *   o While the signal handler does allow contents to be modified, we are
 *     starting with making the interface private and thus allowing us to excise
 *     components that are in their initial state.
 *
 *   o There are similarities to what we want to create with the compressed
 *     xsave format; however, because we don't always have support for the
 *     compressed format, we can't just arbitrarily say let's do a compressed
 *     save to the user stack.
 *
 *   o Because we are not handing this state directly to and from hardware, we
 *     don't need to meet some of the constraints of the compressed xsave format
 *     around wanting alignment for the initial save or additional components.
 *
 * All of the above lead us to our own unique format for this data. When the
 * UC_XSAVE flag is set in the ucontext_t, the uc_xsave member points to a
 * uc_xsave_t structure which has a magic version number, a 32-bit length of the
 * overall structure, and the 64-bit state bit-vector to represent which
 * components are valid. Following this 8-byte header, each component that is
 * present in the bit vector is immediately written out in roughly ascending bit
 * order (the order is determined based on the order of the fpu_xsave_info
 * array).
 *
 * This makes the rough logic that we have here when taking a signal and writing
 * out this state as:
 *
 *   1. Ensure that the FPU is saved and that the contents of the pcb save area
 *      are valid. That is, call fp_save() if the state is not already flagged
 *      with FPU_VALID.
 *
 *   2. Copy the bit-vector from the save area and remove the XFEATURE_LEGACY_FP
 *      and XFEATURE_SSE bits as these will be placed in the xsave area.
 *
 *   3. Initialize the uc_xsave_t by setting our version field, initializing the
 *      length to the length of the current structure, and then setting the
 *      modified bit vector above.
 *
 *   4. Walk each remaining bit of the bit-vector. For each set bit, copy out
 *      its extended state starting at the current length in the header and then
 *      increase the header size by that length.
 *
 *   5. Finally write out the final uc_xsave_t structure.
 *
 * The above process is also used when someone manually calls getcontext_extd(2)
 * to get this state. The main difference between the two is which copyout
 * function we use. This deserves some explanation. Our main starting point for
 * all the logic here is fpu_signal_copyout(). It takes a copyfunc that allows
 * the signal handling context to operate with a different copyout than we
 * normally use in say getcontext_extd(2).
 *
 * When we've received a signal, we're at the intersection of several different
 * gotchas. Normal copyout (or ddi_copyout()) will trigger watchpoints. That is,
 * the watchpoints effectively set a copyout override function (t_copyops) that
 * we end up vectoring to rather than a normal copyout. This allows the data to
 * be modified and for the watchpoint to fire. While this is all well and good
 * normally, it is problematic if we are trying to handle a signal. The signal
 * deliver logic, sendsig(), goes through and disables the watchpoint for the
 * region of the stack that we are copying out to. However, disabling
 * watchpoints is not sufficient, we also need to use the copyout_noerr
 * variants.
 *
 * These variants also require the use of on_fault() and no_fault() for error
 * handling. While it is tempting to try and on_fault() the entire
 * fpu_signal_copyout() operation, that is actually fraught for a few reasons.
 * The first is that we don't want to disable faults during the entire operation
 * as if the kernel messes up we will treat that as a user error. That isn't
 * theoretical and happened during development. The second and perhaps more
 * important issue is that correctly bounding the on_fault() / no_fault() means
 * being careful about state. For example, kernel pre-emption is often disabled
 * during parts of these operations, but it needs to be re-enabled when we're
 * done. This would require tracking in some volatile variable that this had
 * been enabled and disabled and tracking that.
 *
 * Instead, this is why fpu_signal_copyout() takes a copy out function as an
 * argument. When we're in signal handling context, the function will use
 * coypout_noerr() and wrap it in the appropriate on_fault() mechanisms.
 *
 * RESTORING STATE
 *
 * Copying out our current state is the easier half of this problem. When the
 * kernel is done with a signal it calls setcontext(2) with the ucontext_t we
 * assembled for it as described above. setcontext(2) isn't just used for
 * returning from signals.
 *
 * The process for this goes in two steps. The first step is to copy in,
 * validate, and transform the ucontext_t UC_XSAVE that we created above into an
 * equivalent xsave format that we can use the appropriate xrstor function on.
 * This first phase is implemented in fpu_signal_copyin(). Once that is done, we
 * come back through a second phase that is driven out of restorecontext() and
 * is implemented in fpu_set_xsave().
 *
 * Let's start by discussing the second part of this, which is more
 * straightforward. In particular, the second phase assumes that all of the
 * validation and error handling has been done by the first phase. This means
 * here, we have a buffer that is already the appropriate size
 * (cpuid_get_xsave_size()) and all we need to do is make sure that we can
 * replace the actual save state with the current one.
 *
 * The only piece of shenanigans we have to do is around the kernel provided
 * notion of 'status' and 'xstatus', which are cached versions of the x87 and
 * SSE exception vectors. These are part of the fpregset ABI and therefore we
 * need to propagate them from the temporary storage that part 1 sets up in the
 * ignored region of the fxsave data. We use that because it is not persisted by
 * the CPU, so clobbering it is generally alright.
 *
 * Once that is done, we simply note that we need a PCB update to occur to
 * refresh the FPU state before we return to userland. Given that someone has
 * called setcontext(2), this was always going to happen because we have to
 * update segment registers and related, so this isn't so bad. With that, let's
 * move onto the more nuanced part (1).
 *
 * When we're handling a setcontext(2) we have, in userland, a data structure
 * that should match one we serialized out, though we cannot assume that a user
 * has not modified it either accidentally or maliciously. Our goal is to set up
 * the appropriate xsave state that can be passed to the CPU's xrstor. The first
 * problem we have to deal with is where do we actually put this state?
 *
 * While not many programs actually call setcontext(2) of their own volition,
 * this is going to get hit every time we take a signal. The first thought was
 * to re-use the existing thread's save area; however, that's a bit challenging
 * for a few reasons. In particular, we would need to ensure that we don't go
 * off-CPU for any reason, which we cannot assume with a copyin from a user
 * address space. In particular, it is trivial for us to hit a case where the
 * stack has been paged out for some reason, which eschews that path.
 *
 * Instead, whenever a thread first calls setcontext(2), generally from signal
 * context, we will at that time allocate another entry from the 'fpsave_cachep'
 * kmem cache, giving us a buffer of the appropriate space to handle this. Once
 * this buffer has been allocated, we leave it assigned to the thread's pcb and
 * only tear it down when the thread itself finally exits. We reason that a
 * thread that takes a signal once is either going to have the process exit
 * shortly thereafter or is much more likely to take a signal again in the
 * future. Many daemons and other processes set things up so signals are
 * dispatched via one location, masking signals in other thread, using
 * sigsuspend(2), signalfd(3C), or something similar.
 *
 * With this buffer in hand, we begin our task of reassembling state. Note, all
 * of this is conditional on UC_XSAVE being set in the uc_flags member of the
 * ucontext_t. If it is not set, then we assume that there is no extended state
 * and will use the traditional path of setting the fpregset_t into the system
 * via setfpregs().
 *
 * We first will copyin and validate the uc_xsave_t. In particular, we need to
 * make sure the version makes sense, that the xsave component bit-vector
 * doesn't have anything unexpected and more importantly unsupported in it, and
 * that the addresses we've been given are within the user address space. At
 * this point we can walk through our table of implemented bits and process
 * them.
 *
 * For most components in here, the processing is straightforward. We continue
 * walking our cursor and copy data into the kernel and place it in the
 * appropriate place in our xsave state. If a xsave state component bit-vector
 * isn't set, then we must ensure that we have the item in the initial state,
 * which for everything other than the x87/SSE state is the memory being zeroed.
 *
 * The most unique case in the copyin state is that of the x87/SSE state. You
 * might recall that we didn't copy it out explicitly as part of the uc_xsave_t,
 * but instead have opted to use the single definition in the fpregset_t. Thus
 * here, we copy it out of the fpregset_t, which the kernel has helpfully
 * already unified into the 64-bit fxsave version prior to calling us, and
 * install that into the save area we're building up.
 *
 * As part of this, there are two important pieces to be aware of. The first is
 * that because the fpregset_t has both the status and xstatus members
 * mentioned earlier, we temporarily copy them to the software-usable ignored
 * areas of the fxsave state so we can corral this extra state into part (2)
 * without needing to allocate additional space. The second piece is that when
 * we're done processing this we explicitly remove the UC_FPU flag that would
 * tell the kernel to proceed with updating that region. The problem is that
 * that goes directly into the pcb's save area and not to the intermediate
 * buffer as it uses the same entry point as /proc, mainly setfpregs().
 *
 * We don't do much validation of the actual contents of the registers that are
 * being set with the exception of ensuring that no reserved bits of the mxcsr
 * are used. This is not as strict as /proc, but failure here means the process
 * is likely going to die (returning from setcontext() in a signal handler is
 * fatal).
 *
 * /proc xregs
 * -----------
 *
 * Observability of the state of the extended registers is important for
 * understanding the system. While on the surface this is similar to signal
 * handling, it is crucially different in a number of ways:
 *
 *   o In signal handling, we're trying to conserve every byte of stack that we
 *     can.
 *   o The /proc xregs file will end up in core files, which means that we need
 *     a way of knowing what components are present and not present in it,
 *     because this will vary from CPU to CPU due to the addition of
 *     architectural features. For example, some CPUs support AVX-512, but
 *     others do not.
 *
 *   o The signal handling structure (uc_xsave_t) is private and we're not
 *     trying to have software modify it, on the other hand, the /proc
 *     interfaces that we support we do want software to be able to interrogate
 *     and manipulate. These need to be something that we can introduce
 *     additional components into and make other changes that still allow it to
 *     work.
 *
 * The x86 xregs format is documented in proc(5). The short form is that the
 * prxregset_hdr_t has a number of information entries, which are of the type
 * prxregset_info_t. Each of the information headers has a type, size, and
 * offset which indicate where to find the additional data.
 *
 * Each entry is described as one of the entries in the fpu_xsave_info[]. These
 * items either are a 1:1 correspondence with a xsave related feature (e.g.
 * there is one entry for each of the three AVX-512 components) or it is
 * something synthetic that we provide as additional information such as the
 * PRX_INFO_XCR, which is a way of getting information about the system such as
 * what is enabled in %xcr0 out there.
 *
 * Unlike signal handling, we are given the buffer to place everything that
 * needs to be written out. This is partially the design of the /proc APIs. That
 * is, we will always assemble everything into the entire buffer that /proc asks
 * us to, and then it will use as much or as little of it as is required.
 * Similarly, when setting things, we don't have to worry about copying in
 * information in the same way as signal handling does, because /proc takes care
 * of it and always hands us a full buffer. Sizing that is a little nuanced, but
 * is all handled in prmachdep.c.
 *
 * When someone performs a read of the xregs and thus is asking us for the
 * current state, there is a little bit of nuance that we need to deal with.
 * The first, is whether or not the FPU is enabled and the second is if the FPU
 * is enabled, whether a given component is noted as being in its initial state.
 * This basically gives us three possible states for a given component:
 *
 *   1. FPU_EN is not set and FPU_VALID is not set. This means we need to take
 *      the illumos FPU default for an item. More on that in a moment.
 *   2. The saved xsave state indicates that the bit for a given component is
 *      zero -- specifically the xsh_xstate_bv member of the struct xsave_state.
 *      In this case, we must take the CPU's default for an item. This is
 *      usually the same as illumos, but not always.
 *   3. The saved xsave state indicates that a given component's state bit is
 *      valid. The simplest of our cases. We can just take what we have from the
 *      xsave state.
 *
 * The CPU's default state for most components other than the x87/SSE state is
 * to have it be zeroed. This is what we treat as our default state as well. The
 * primary difference is in the initialization of the x87/SSE state. The SYS V
 * ABI requires that we enable a different floating point control word then the
 * hardware default. This means that when we're dealing with case (1) for
 * x87/SSE we have to be more careful than the other components. Thankfully for
 * everything else this is just keeping it zeroed.
 *
 * A reasonable question would be why not just skip components that aren't
 * marked as present. There are a few reasons we take a different approach and
 * always include them. Both of these are to make lives simpler for consumers.
 * In the first case, when someone is performing a read and wants to reassemble
 * and answer the question of 'what is the value of %ymm0 or %zmm15', they have
 * to combine multiple disparate parts. If one knows that the data we put into
 * there is always valid and represents what is in hardware and doesn't have to
 * keep track of what are the defaults in different circumstances, then that
 * greatly simplifies consumers lives. It also helps us for core files and other
 * observability cases because the answer to what is the operating system's
 * default may change over time.
 *
 * Similarly, including all the possible structures means that we have
 * simplified writes. Writes are always setting the full state of a thread,
 * meaning that if someone wants to modify only a single register they must do a
 * read, modify, and write. By including everything that they might need, it
 * makes it easier for consumers to do this and not have to cons up the whole
 * structure on their own.
 *
 * When we're setting state, things change around a little bit. We have a few
 * constraints that are laid out in proc(5). In particular, we require that the
 * PRX_INFO_XSAVE component always be present to tell us which other components
 * we expect to be here and which ones we don't. We also are much stricter about
 * writes in several ways. Of all the components, the PRX_INFO_XCR is read-only
 * and may not be modified by a calling process. In addition, when we have
 * 32-bit applications which have reserved registers in the %ymm, %zmm, etc.
 * components, if they are being written to and have modifications, then we will
 * indicate an error there.
 *
 * Because we are given the entire buffer from userland and don't need to have
 * an intermediate place to copy it in, we will validate the entire thing in
 * advance. Once it has been validated and we consider it legal, then we will
 * translate each entry into its corresponding entry in pcb's normal floating
 * point state. This is different from signal handling mostly because of the
 * fact that we are not using copyin, and once we get to this point, there is
 * no more validation, so we don't have the same concerns around blocking while
 * pre-emption is disabled.
 *
 * The Wrinkle with fpregs
 * -----------------------
 *
 * When we instead turn our attention to the fpregs, whether we're gathering
 * them as part of the ucontext_t or as part of /proc, there are a few
 * complications that we need to be aware of when we're operating on a kernel
 * that is using xsave as the save mechanism. When we're using fxsave as the
 * save mechanism, the CPU will always save the entire 512-byte fxsave region.
 * The fpregs ABI that the kernel expects is basically this structure itself,
 * which is transformed into a 32-bit compatible form in archdep.c.
 *
 * But xsave makes this much more complex and has historically been a source of
 * bugs in the system. In particular, unlike fxsave, xsave has its component bit
 * vector that is written out to indicate validity. This means that blindly
 * copying the fxsave area without checking those bits will lead us to do the
 * wrong thing. The XMM state flag mostly covers the 16 128-bit %xmm registers,
 * while the x87 legacy fp flag covers the rest of the state. This is all good,
 * aside from the MCXSR.
 *
 * One of the more complicated pieces of xsave state management is correctly
 * answering the question of when the MXCSR is written out to xsave_state. In
 * practice, this is rather convoluted and varies. If either the XMM or AVX
 * feature bits are set then the CPU will write out the MXCSR and its mask
 * register into the traditional fxsave state region. This behavior is dependent
 * on the type of save function that we use. xsave and xsaveopt will look at the
 * AVX feature bit; however, xsavec does not and only considers the SSE feature
 * bit. This means that when we're retrieving things, we need to check both of
 * those bits to determine if we should use the initial state or the value
 * written out.
 *
 * When we come to someone trying to set the fpregs through /proc, the main
 * question we have is what happens to the extended registers. We have opted to
 * implement and document it such that a write to the fpregs only impacts the
 * fpregs. Put differently, we will save the FPU state with fp_save() ahead of
 * copying the data into the save area, set the state bits for x87 and XMM
 * state, and then set the FPU to be restored. All in all, this basically means
 * that writing to fpregs does not touch any of the %ymm, %zmm, or other state
 * that we might have present.
 *
 * Forward Looking: Adding Intel AMX Support
 * -----------------------------------------
 *
 * Nothing can stop the march of features being added into the FPU. One of the
 * larger chunks that we will need to wrangle with is Intel's Advanced Matrix
 * Extensions (AMX), which add a large chunk of xsave state to each process.
 * While things like AVX and AVX-512 have been enabled by default, the broader
 * OS community has not been wanting to do this for AMX ,because of the size of
 * the state which exceeds 8 KiB. While the signal handling state went out of
 * its way to minimize the size it wrote to the stack, if this is used, it would
 * need to be preserved.
 *
 * To deal with this reality and the fact that folks don't really want to
 * enable it by default for all purposes when its use will be quite special
 * purpose, Intel has also added a MSR around extended feature disable or xfd.
 * This is what we represent in the PRX_INFO_XCR prx_xfd member. Our starting
 * assumption, and the reason that so much of the /proc and signal logic ensures
 * that we have the thread and process around, taking as an example the unused
 * process argument in fpu_proc_xregs_info(), is that we will follow suit and
 * default to having support disabled, but that a process will be able to opt
 * into it, which will result in several different assumptions around signal
 * stack sizing and cause us to reallocate and extend the pcb's FPU save state.
 *
 * The following is a list of items to pay attention to for future folks who
 * work on this:
 *
 *   o We will want to confirm whether other systems have opted to make this
 *     process-wide or thread-wide. Assuming process-wide, we will need to do a
 *     hold of all lwps while making a change. The interface for that probably
 *     doesn't want to be /proc, as a process probably doesn't want to write to
 *     its own control file. Changing it for another process could be done
 *     through the agent-lwp.
 *   o Opting into this should probably be a one-way street.
 *   o Opting into this will need to evaluate all threads and in particular
 *     stack sizes to confirm they adhere to the new minimum.
 *   o We will need to make sure that setting and clearing the xfd MSR is part
 *     of the FPU context ops and something we set by default on every CPU.
 *   o We will need to add a new interface to allow opting into this feature.
 *   o We will need to ensure that all subsequently created signal stacks adhere
 *     to a required minimum size that we communicate through libc.
 *   o We will need to make sure that both rtld and libc no longer rely on a
 *     static value of the AT_SUN_FPSIZE, but rather realize that this can be
 *     dynamic. At that time, we should evaluate if we can get away with not
 *     needing to save this for rtld, even though signal handlers should assume
 *     they will.
 *   o The various components (because there is more than one) will want to be
 *     added to the fpu_xsave_info[]. Consulting the processes's xfd will be
 *     required and probably require logic changes.
 *
 * The above is not exhaustive. We'll probably have some other issues and fun
 * while doing this.
 */

/*
 * The kind of FPU we advertise to rtld so it knows what to do when working
 * through the PLT.
 */
int fp_elf = AT_386_FPINFO_FXSAVE;

/*
 * Mechanism to save FPU state.
 */
int fp_save_mech = FP_FXSAVE;

/*
 * See section 10.5.1 in the Intel 64 and IA-32 Architectures Software
 * Developer's Manual, Volume 1.
 */
#define	FXSAVE_ALIGN	16

/*
 * See section 13.4 in the Intel 64 and IA-32 Architectures Software
 * Developer's Manual, Volume 1.
 */
#define	XSAVE_ALIGN	64

kmem_cache_t *fpsave_cachep;

/* Legacy fxsave layout + xsave header + ymm */
#define	AVX_XSAVE_SIZE		(512 + 64 + 256)

/*
 * Various sanity checks.
 */
CTASSERT(sizeof (struct fxsave_state) == 512);
CTASSERT(sizeof (struct fnsave_state) == 108);
CTASSERT((offsetof(struct fxsave_state, fx_xmm[0]) & 0xf) == 0);
CTASSERT(sizeof (struct xsave_state) >= AVX_XSAVE_SIZE);

/*
 * Basic architectural alignment information.
 */
#define	FPU_ALIGN_XMM	16
#define	FPU_ALIGN_YMM	32
#define	FPU_ALIGN_ZMM	64

/*
 * This structure is the x86 implementation of the kernel FPU that is defined in
 * uts/common/sys/kfpu.h.
 */

typedef enum kfpu_flags {
	/*
	 * This indicates that the save state has initial FPU data.
	 */
	KFPU_F_INITIALIZED = 0x01
} kfpu_flags_t;

struct kfpu_state {
	fpu_ctx_t	kfpu_ctx;
	kfpu_flags_t	kfpu_flags;
	kthread_t	*kfpu_curthread;
};

/*
 * Initial kfpu state for SSE/SSE2 used by fpinit()
 */
const struct fxsave_state sse_initial = {
	FPU_CW_INIT,	/* fx_fcw */
	0,		/* fx_fsw */
	0,		/* fx_fctw */
	0,		/* fx_fop */
	0,		/* fx_rip */
	0,		/* fx_rdp */
	SSE_MXCSR_INIT	/* fx_mxcsr */
	/* rest of structure is zero */
};

/*
 * Initial kfpu state for AVX used by fpinit()
 */
const struct xsave_state avx_initial = {
	/*
	 * The definition below needs to be identical with sse_initial
	 * defined above.
	 */
	.xs_fxsave = {
		.fx_fcw = FPU_CW_INIT,
		.fx_mxcsr = SSE_MXCSR_INIT,
	},
	.xs_header = {
		/*
		 * bit0 = 1 for XSTATE_BV to indicate that legacy fields are
		 * valid, and CPU should initialize XMM/YMM.
		 */
		.xsh_xstate_bv = 1,
		.xsh_xcomp_bv = 0,
	},
};

/*
 * mxcsr_mask value (possibly reset in fpu_probe); used to avoid
 * the #gp exception caused by setting unsupported bits in the
 * MXCSR register
 */
uint32_t sse_mxcsr_mask = SSE_MXCSR_MASK_DEFAULT;

/*
 * This vector is patched to xsave_ctxt() or xsaveopt_ctxt() if we discover we
 * have an XSAVE-capable chip in fpu_probe.
 */
void (*fpsave_ctxt)(void *) = fpxsave_ctxt;
void (*fprestore_ctxt)(void *) = fpxrestore_ctxt;

/*
 * This function pointer is changed to xsaveopt if the CPU is xsaveopt capable.
 */
void (*xsavep)(struct xsave_state *, uint64_t) = xsave;

static int fpe_sicode(uint_t);
static int fpe_simd_sicode(uint_t);
static void fp_new_lwp(void *, void *);
static void fp_free_ctx(void *, int);

static struct ctxop *
fp_ctxop_allocate(struct fpu_ctx *fp)
{
	const struct ctxop_template tpl = {
		.ct_rev		= CTXOP_TPL_REV,
		.ct_save	= fpsave_ctxt,
		.ct_restore	= fprestore_ctxt,
		.ct_fork	= fp_new_lwp,
		.ct_lwp_create	= fp_new_lwp,
		.ct_free	= fp_free_ctx,
	};
	return (ctxop_allocate(&tpl, fp));
}

/*
 * Copy the state of parent lwp's floating point context into the new lwp.
 * Invoked for both fork() and lwp_create().
 *
 * Note that we inherit -only- the control state (e.g. exception masks,
 * rounding, precision control, etc.); the FPU registers are otherwise
 * reset to their initial state.
 */
static void
fp_new_lwp(void *parent, void *child)
{
	kthread_id_t t = parent, ct = child;
	struct fpu_ctx *fp;		/* parent fpu context */
	struct fpu_ctx *cfp;		/* new fpu context */
	struct fxsave_state *fx, *cfx;
	struct xsave_state *cxs;

	ASSERT(fp_kind != FP_NO);

	fp = &t->t_lwp->lwp_pcb.pcb_fpu;
	cfp = &ct->t_lwp->lwp_pcb.pcb_fpu;

	/*
	 * If the parent FPU state is still in the FPU hw then save it;
	 * conveniently, fp_save() already does this for us nicely.
	 */
	fp_save(fp);

	cfp->fpu_flags = FPU_EN | FPU_VALID;
	cfp->fpu_regs.kfpu_status = 0;
	cfp->fpu_regs.kfpu_xstatus = 0;

	/*
	 * Make sure that the child's FPU is cleaned up and made ready for user
	 * land.
	 */
	PCB_SET_UPDATE_FPU(&ct->t_lwp->lwp_pcb);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fx = fp->fpu_regs.kfpu_u.kfpu_fx;
		cfx = cfp->fpu_regs.kfpu_u.kfpu_fx;
		bcopy(&sse_initial, cfx, sizeof (*cfx));
		cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
		cfx->fx_fcw = fx->fx_fcw;
		break;

	case FP_XSAVE:
		cfp->fpu_xsave_mask = fp->fpu_xsave_mask;

		VERIFY(fp->fpu_regs.kfpu_u.kfpu_xs != NULL);

		fx = &fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave;
		cxs = cfp->fpu_regs.kfpu_u.kfpu_xs;
		cfx = &cxs->xs_fxsave;

		bcopy(&avx_initial, cxs, sizeof (*cxs));
		cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
		cfx->fx_fcw = fx->fx_fcw;
		cxs->xs_header.xsh_xstate_bv |=
		    (get_xcr(XFEATURE_ENABLED_MASK) & XFEATURE_FP_INITIAL);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	/*
	 * Mark that both the parent and child need to have the FPU cleaned up
	 * before returning to userland.
	 */

	ctxop_attach(ct, fp_ctxop_allocate(cfp));
}

/*
 * Free any state associated with floating point context.
 * Fp_free can be called in three cases:
 * 1) from reaper -> thread_free -> freectx-> fp_free
 *	fp context belongs to a thread on deathrow
 *	nothing to do,  thread will never be resumed
 *	thread calling ctxfree is reaper
 *
 * 2) from exec -> freectx -> fp_free
 *	fp context belongs to the current thread
 *	must disable fpu, thread calling ctxfree is curthread
 *
 * 3) from restorecontext -> setfpregs -> fp_free
 *	we have a modified context in the memory (lwp->pcb_fpu)
 *	disable fpu and release the fp context for the CPU
 *
 */
void
fp_free(struct fpu_ctx *fp)
{
	ASSERT(fp_kind != FP_NO);

	if (fp->fpu_flags & FPU_VALID)
		return;

	kpreempt_disable();
	/*
	 * We want to do fpsave rather than fpdisable so that we can
	 * keep the fpu_flags as FPU_VALID tracking the CR0_TS bit
	 */
	fp->fpu_flags |= FPU_VALID;
	/* If for current thread disable FP to track FPU_VALID */
	if (curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu) {
		/* Clear errors if any to prevent frstor from complaining */
		(void) fperr_reset();
		if (fp_kind & __FP_SSE)
			(void) fpxerr_reset();
		fpdisable();
	}
	kpreempt_enable();
}

/*
 * Wrapper for freectx to make the types line up for fp_free()
 */
static void
fp_free_ctx(void *arg, int isexec __unused)
{
	fp_free((struct fpu_ctx *)arg);
}

/*
 * Store the floating point state and disable the floating point unit.
 */
void
fp_save(struct fpu_ctx *fp)
{
	ASSERT(fp_kind != FP_NO);

	kpreempt_disable();
	if (!fp || fp->fpu_flags & FPU_VALID ||
	    (fp->fpu_flags & FPU_EN) == 0) {
		kpreempt_enable();
		return;
	}
	ASSERT(curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpxsave(fp->fpu_regs.kfpu_u.kfpu_fx);
		break;

	case FP_XSAVE:
		xsavep(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_flags |= FPU_VALID;

	/*
	 * We save the FPU as part of forking, execing, modifications via /proc,
	 * restorecontext, etc. As such, we need to make sure that we return to
	 * userland with valid state in the FPU. If we're context switched out
	 * before we hit sys_rtt_common() we'll end up having restored the FPU
	 * as part of the context ops operations. The restore logic always makes
	 * sure that FPU_VALID is set before doing a restore so we don't restore
	 * it a second time.
	 */
	PCB_SET_UPDATE_FPU(&curthread->t_lwp->lwp_pcb);

	kpreempt_enable();
}

/*
 * Restore the FPU context for the thread:
 * The possibilities are:
 *	1. No active FPU context: Load the new context into the FPU hw
 *	   and enable the FPU.
 */
void
fp_restore(struct fpu_ctx *fp)
{
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpxrestore(fp->fpu_regs.kfpu_u.kfpu_fx);
		break;

	case FP_XSAVE:
		xrestore(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_flags &= ~FPU_VALID;
}

/*
 * Reset the FPU such that it is in a valid state for a new thread that is
 * coming out of exec. The FPU will be in a usable state at this point. At this
 * point we know that the FPU state has already been allocated and if this
 * wasn't an init process, then it will have had fp_free() previously called.
 */
void
fp_exec(void)
{
	struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	if (fp_save_mech == FP_XSAVE) {
		fp->fpu_xsave_mask = XFEATURE_FP_ALL;
	}

	struct ctxop *ctx = fp_ctxop_allocate(fp);
	/*
	 * Make sure that we're not preempted in the middle of initializing the
	 * FPU on CPU.
	 */
	kpreempt_disable();
	ctxop_attach(curthread, ctx);
	fpinit();
	fp->fpu_flags = FPU_EN;
	kpreempt_enable();
}


/*
 * Seeds the initial state for the current thread.  The possibilities are:
 *      1. Another process has modified the FPU state before we have done any
 *         initialization: Load the FPU state from the LWP state.
 *      2. The FPU state has not been externally modified:  Load a clean state.
 */
void
fp_seed(void)
{
	struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(curthread->t_preempt >= 1);
	ASSERT((fp->fpu_flags & FPU_EN) == 0);

	/*
	 * Always initialize a new context and initialize the hardware.
	 */
	if (fp_save_mech == FP_XSAVE) {
		fp->fpu_xsave_mask = XFEATURE_FP_ALL;
	}

	ctxop_attach(curthread, fp_ctxop_allocate(fp));
	fpinit();

	/*
	 * If FPU_VALID is set, it means someone has modified registers via
	 * /proc.  In this case, restore the current lwp's state.
	 */
	if (fp->fpu_flags & FPU_VALID)
		fp_restore(fp);

	ASSERT((fp->fpu_flags & FPU_VALID) == 0);
	fp->fpu_flags = FPU_EN;
}

/*
 * When using xsave/xrstor, these three functions are used by the lwp code to
 * manage the memory for the xsave area.
 */
void
fp_lwp_init(klwp_t *lwp)
{
	struct fpu_ctx *fp = &lwp->lwp_pcb.pcb_fpu;

	/*
	 * We keep a copy of the pointer in lwp_fpu so that we can restore the
	 * value in forklwp() after we duplicate the parent's LWP state.
	 */
	lwp->lwp_fpu = fp->fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, KM_SLEEP);
	fp->fpu_signal = NULL;

	if (fp_save_mech == FP_XSAVE) {
		/*
		 *
		 * We bzero since the fpinit() code path will only
		 * partially initialize the xsave area using avx_inital.
		 */
		ASSERT(cpuid_get_xsave_size() >= sizeof (struct xsave_state));
		bzero(fp->fpu_regs.kfpu_u.kfpu_xs, cpuid_get_xsave_size());
	}
}

void
fp_lwp_cleanup(klwp_t *lwp)
{
	struct fpu_ctx *fp = &lwp->lwp_pcb.pcb_fpu;

	if (fp->fpu_regs.kfpu_u.kfpu_generic != NULL) {
		kmem_cache_free(fpsave_cachep,
		    fp->fpu_regs.kfpu_u.kfpu_generic);
		lwp->lwp_fpu = fp->fpu_regs.kfpu_u.kfpu_generic = NULL;
	}

	if (fp->fpu_signal != NULL) {
		kmem_cache_free(fpsave_cachep, fp->fpu_signal);
		fp->fpu_signal = NULL;
	}
}

/*
 * Called during the process of forklwp(). The kfpu_u pointer will have been
 * overwritten while copying the parent's LWP structure. We have a valid copy
 * stashed in the child's lwp_fpu which we use to restore the correct value.
 */
void
fp_lwp_dup(klwp_t *lwp)
{
	void *xp = lwp->lwp_fpu;
	size_t sz;

	switch (fp_save_mech) {
	case FP_FXSAVE:
		sz = sizeof (struct fxsave_state);
		break;
	case FP_XSAVE:
		sz = cpuid_get_xsave_size();
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	/* copy the parent's values into the new lwp's struct */
	bcopy(lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic, xp, sz);
	/* now restore the pointer */
	lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic = xp;
	/* Ensure that we don't inherit our parent's signal state */
	lwp->lwp_pcb.pcb_fpu.fpu_signal = NULL;
}

/*
 * Handle a processor extension error fault
 * Returns non zero for error.
 */

/*ARGSUSED*/
int
fpexterrflt(struct regs *rp)
{
	uint32_t fpcw, fpsw;
	fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(fp_kind != FP_NO);

	/*
	 * Now we can enable the interrupts.
	 * (NOTE: x87 fp exceptions come thru interrupt gate)
	 */
	sti();

	if (!fpu_exists)
		return (FPE_FLTINV);

	/*
	 * Do an unconditional save of the FP state.  If it's dirty (TS=0),
	 * it'll be saved into the fpu context area passed in (that of the
	 * current thread).  If it's not dirty (it may not be, due to
	 * an intervening save due to a context switch between the sti(),
	 * above and here, then it's safe to just use the stored values in
	 * the context save area to determine the cause of the fault.
	 */
	fp_save(fp);

	/* clear exception flags in saved state, as if by fnclex */
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpsw = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw;
		fpcw = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fcw;
		fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw &= ~FPS_SW_EFLAGS;
		break;

	case FP_XSAVE:
		fpsw = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw;
		fpcw = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fcw;
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw &= ~FPS_SW_EFLAGS;
		/*
		 * Always set LEGACY_FP as it may have been cleared by XSAVE
		 * instruction
		 */
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP;
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_regs.kfpu_status = fpsw;

	if ((fpsw & FPS_ES) == 0)
		return (0);		/* No exception */

	/*
	 * "and" the exception flags with the complement of the mask
	 * bits to determine which exception occurred
	 */
	return (fpe_sicode(fpsw & ~fpcw & 0x3f));
}

/*
 * Handle an SSE/SSE2 precise exception.
 * Returns a non-zero sicode for error.
 */
/*ARGSUSED*/
int
fpsimderrflt(struct regs *rp)
{
	uint32_t mxcsr, xmask;
	fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(fp_kind & __FP_SSE);

	/*
	 * NOTE: Interrupts are disabled during execution of this
	 * function.  They are enabled by the caller in trap.c.
	 */

	/*
	 * The only way we could have gotten here if there is no FP unit
	 * is via a user executing an INT $19 instruction, so there is
	 * no fault in that case.
	 */
	if (!fpu_exists)
		return (0);

	/*
	 * Do an unconditional save of the FP state.  If it's dirty (TS=0),
	 * it'll be saved into the fpu context area passed in (that of the
	 * current thread).  If it's not dirty, then it's safe to just use
	 * the stored values in the context save area to determine the
	 * cause of the fault.
	 */
	fp_save(fp);		/* save the FPU state */

	if (fp_save_mech == FP_XSAVE) {
		mxcsr = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_mxcsr;
		fp->fpu_regs.kfpu_status =
		    fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw;
	} else {
		mxcsr = fp->fpu_regs.kfpu_u.kfpu_fx->fx_mxcsr;
		fp->fpu_regs.kfpu_status = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw;
	}
	fp->fpu_regs.kfpu_xstatus = mxcsr;

	/*
	 * compute the mask that determines which conditions can cause
	 * a #xm exception, and use this to clean the status bits so that
	 * we can identify the true cause of this one.
	 */
	xmask = (mxcsr >> 7) & SSE_MXCSR_EFLAGS;
	return (fpe_simd_sicode((mxcsr & SSE_MXCSR_EFLAGS) & ~xmask));
}

/*
 * In the unlikely event that someone is relying on this subcode being
 * FPE_FLTILL for denormalize exceptions, it can always be patched back
 * again to restore old behaviour.
 */
int fpe_fltden = FPE_FLTDEN;

/*
 * Map from the FPU status word to the FP exception si_code.
 */
static int
fpe_sicode(uint_t sw)
{
	if (sw & FPS_IE)
		return (FPE_FLTINV);
	if (sw & FPS_ZE)
		return (FPE_FLTDIV);
	if (sw & FPS_DE)
		return (fpe_fltden);
	if (sw & FPS_OE)
		return (FPE_FLTOVF);
	if (sw & FPS_UE)
		return (FPE_FLTUND);
	if (sw & FPS_PE)
		return (FPE_FLTRES);
	return (FPE_FLTINV);	/* default si_code for other exceptions */
}

/*
 * Map from the SSE status word to the FP exception si_code.
 */
static int
fpe_simd_sicode(uint_t sw)
{
	if (sw & SSE_IE)
		return (FPE_FLTINV);
	if (sw & SSE_ZE)
		return (FPE_FLTDIV);
	if (sw & SSE_DE)
		return (FPE_FLTDEN);
	if (sw & SSE_OE)
		return (FPE_FLTOVF);
	if (sw & SSE_UE)
		return (FPE_FLTUND);
	if (sw & SSE_PE)
		return (FPE_FLTRES);
	return (FPE_FLTINV);	/* default si_code for other exceptions */
}

/*
 * This routine is invoked as part of libc's __fpstart implementation
 * via sysi86(2).
 *
 * It may be called -before- any context has been assigned in which case
 * we try and avoid touching the hardware.  Or it may be invoked well
 * after the context has been assigned and fiddled with, in which case
 * just tweak it directly.
 */
void
fpsetcw(uint16_t fcw, uint32_t mxcsr)
{
	struct fpu_ctx *fp = &curthread->t_lwp->lwp_pcb.pcb_fpu;
	struct fxsave_state *fx;

	if (!fpu_exists || fp_kind == FP_NO)
		return;

	if ((fp->fpu_flags & FPU_EN) == 0) {
		if (fcw == FPU_CW_INIT && mxcsr == SSE_MXCSR_INIT) {
			/*
			 * Common case.  Floating point unit not yet
			 * enabled, and kernel already intends to initialize
			 * the hardware the way the caller wants.
			 */
			return;
		}
		/*
		 * Hmm.  Userland wants a different default.
		 * Do a fake "first trap" to establish the context, then
		 * handle as if we already had a context before we came in.
		 */
		kpreempt_disable();
		fp_seed();
		kpreempt_enable();
	}

	/*
	 * Ensure that the current hardware state is flushed back to the
	 * pcb, then modify that copy.  Next use of the fp will
	 * restore the context.
	 */
	fp_save(fp);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fx = fp->fpu_regs.kfpu_u.kfpu_fx;
		fx->fx_fcw = fcw;
		fx->fx_mxcsr = sse_mxcsr_mask & mxcsr;
		break;

	case FP_XSAVE:
		fx = &fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave;
		fx->fx_fcw = fcw;
		fx->fx_mxcsr = sse_mxcsr_mask & mxcsr;
		/*
		 * Always set LEGACY_FP as it may have been cleared by XSAVE
		 * instruction
		 */
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP;
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}
}

static void
kernel_fpu_fpstate_init(kfpu_state_t *kfpu)
{
	struct xsave_state *xs;

	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(&sse_initial, kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_fx,
		    sizeof (struct fxsave_state));
		kfpu->kfpu_ctx.fpu_xsave_mask = 0;
		break;
	case FP_XSAVE:
		xs = kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_xs;
		bzero(xs, cpuid_get_xsave_size());
		bcopy(&avx_initial, xs, sizeof (*xs));
		xs->xs_header.xsh_xstate_bv = XFEATURE_LEGACY_FP | XFEATURE_SSE;
		kfpu->kfpu_ctx.fpu_xsave_mask = XFEATURE_FP_ALL;
		break;
	default:
		panic("invalid fp_save_mech");
	}

	/*
	 * Set the corresponding flags that the system expects on the FPU state
	 * to indicate that this is our state. The FPU_EN flag is required to
	 * indicate that FPU usage is allowed. The FPU_KERN flag is explicitly
	 * not set below as it represents that this state is being suppressed
	 * by the kernel.
	 */
	kfpu->kfpu_ctx.fpu_flags = FPU_EN | FPU_VALID;
	kfpu->kfpu_flags |= KFPU_F_INITIALIZED;
}

kfpu_state_t *
kernel_fpu_alloc(int kmflags)
{
	kfpu_state_t *kfpu;

	if ((kfpu = kmem_zalloc(sizeof (kfpu_state_t), kmflags)) == NULL) {
		return (NULL);
	}

	kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, kmflags);
	if (kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic == NULL) {
		kmem_free(kfpu, sizeof (kfpu_state_t));
		return (NULL);
	}

	kernel_fpu_fpstate_init(kfpu);

	return (kfpu);
}

void
kernel_fpu_free(kfpu_state_t *kfpu)
{
	kmem_cache_free(fpsave_cachep,
	    kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic);
	kmem_free(kfpu, sizeof (kfpu_state_t));
}

static void
kernel_fpu_ctx_save(void *arg)
{
	kfpu_state_t *kfpu = arg;
	fpu_ctx_t *pf;

	if (kfpu == NULL) {
		/*
		 * A NULL kfpu implies this is a kernel thread with an LWP and
		 * no user-level FPU usage. Use the lwp fpu save area.
		 */
		pf = &curthread->t_lwp->lwp_pcb.pcb_fpu;

		ASSERT(curthread->t_procp->p_flag & SSYS);
		ASSERT3U(pf->fpu_flags & FPU_VALID, ==, 0);

		fp_save(pf);
	} else {
		pf = &kfpu->kfpu_ctx;

		ASSERT3P(kfpu->kfpu_curthread, ==, curthread);
		ASSERT3U(pf->fpu_flags & FPU_VALID, ==, 0);

		/*
		 * Note, we can't use fp_save because it assumes that we're
		 * saving to the thread's PCB and not somewhere else. Because
		 * this is a different FPU context, we instead have to do this
		 * ourselves.
		 */
		switch (fp_save_mech) {
		case FP_FXSAVE:
			fpxsave(pf->fpu_regs.kfpu_u.kfpu_fx);
			break;
		case FP_XSAVE:
			xsavep(pf->fpu_regs.kfpu_u.kfpu_xs, pf->fpu_xsave_mask);
			break;
		default:
			panic("Invalid fp_save_mech");
		}

		/*
		 * Because we have saved context here, our save state is no
		 * longer valid and therefore needs to be reinitialized.
		 */
		kfpu->kfpu_flags &= ~KFPU_F_INITIALIZED;
	}

	pf->fpu_flags |= FPU_VALID;

	/*
	 * Clear KFPU flag. This allows swtch to check for improper kernel
	 * usage of the FPU (i.e. switching to a new thread while the old
	 * thread was in the kernel and using the FPU, but did not perform a
	 * context save).
	 */
	curthread->t_flag &= ~T_KFPU;
}

static void
kernel_fpu_ctx_restore(void *arg)
{
	kfpu_state_t *kfpu = arg;
	fpu_ctx_t *pf;

	if (kfpu == NULL) {
		/*
		 * A NULL kfpu implies this is a kernel thread with an LWP and
		 * no user-level FPU usage. Use the lwp fpu save area.
		 */
		pf = &curthread->t_lwp->lwp_pcb.pcb_fpu;

		ASSERT(curthread->t_procp->p_flag & SSYS);
		ASSERT3U(pf->fpu_flags & FPU_VALID, !=, 0);
	} else {
		pf = &kfpu->kfpu_ctx;

		ASSERT3P(kfpu->kfpu_curthread, ==, curthread);
		ASSERT3U(pf->fpu_flags & FPU_VALID, !=, 0);
	}

	fp_restore(pf);
	curthread->t_flag |= T_KFPU;
}

/*
 * Validate that the thread is not switching off-cpu while actively using the
 * FPU within the kernel.
 */
void
kernel_fpu_no_swtch(void)
{
	if ((curthread->t_flag & T_KFPU) != 0) {
		panic("curthread swtch-ing while the kernel is using the FPU");
	}
}

static const struct ctxop_template kfpu_ctxop_tpl = {
	.ct_rev		= CTXOP_TPL_REV,
	.ct_save	= kernel_fpu_ctx_save,
	.ct_restore	= kernel_fpu_ctx_restore,
};

void
kernel_fpu_begin(kfpu_state_t *kfpu, uint_t flags)
{
	klwp_t *pl = curthread->t_lwp;
	struct ctxop *ctx;

	if ((curthread->t_flag & T_KFPU) != 0) {
		panic("curthread attempting to nest kernel FPU states");
	}

	/* KFPU_USE_LWP and KFPU_NO_STATE are mutually exclusive. */
	ASSERT((flags & (KFPU_USE_LWP | KFPU_NO_STATE)) !=
	    (KFPU_USE_LWP | KFPU_NO_STATE));

	if ((flags & KFPU_NO_STATE) == KFPU_NO_STATE) {
		/*
		 * Since we don't have a kfpu_state or usable lwp pcb_fpu to
		 * hold our kernel FPU context, we depend on the caller doing
		 * kpreempt_disable for the duration of our FPU usage. This
		 * should only be done for very short periods of time.
		 */
		ASSERT(curthread->t_preempt > 0);
		ASSERT(kfpu == NULL);

		if (pl != NULL) {
			/*
			 * We might have already saved once so FPU_VALID could
			 * be set. This is handled in fp_save.
			 */
			fp_save(&pl->lwp_pcb.pcb_fpu);
			pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
		}

		curthread->t_flag |= T_KFPU;

		/* Always restore the fpu to the initial state. */
		fpinit();

		return;
	}

	/*
	 * We either have a kfpu, or are using the LWP pcb_fpu for context ops.
	 */

	if ((flags & KFPU_USE_LWP) == 0) {
		if (kfpu->kfpu_curthread != NULL)
			panic("attempting to reuse kernel FPU state at %p when "
			    "another thread already is using", kfpu);

		if ((kfpu->kfpu_flags & KFPU_F_INITIALIZED) == 0)
			kernel_fpu_fpstate_init(kfpu);

		kfpu->kfpu_curthread = curthread;
	}

	/*
	 * Not all threads may have an active LWP. If they do and we're not
	 * going to re-use the LWP, then we should go ahead and save the state.
	 * We must also note that the fpu is now being used by the kernel and
	 * therefore we do not want to manage the fpu state via the user-level
	 * thread's context handlers.
	 *
	 * We might have already saved once (due to a prior use of the kernel
	 * FPU or another code path) so FPU_VALID could be set. This is handled
	 * by fp_save, as is the FPU_EN check.
	 */
	ctx = ctxop_allocate(&kfpu_ctxop_tpl, kfpu);
	kpreempt_disable();
	if (pl != NULL) {
		if ((flags & KFPU_USE_LWP) == 0)
			fp_save(&pl->lwp_pcb.pcb_fpu);
		pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
	}

	/*
	 * Set the context operations for kernel FPU usage.  Because kernel FPU
	 * setup and ctxop attachment needs to happen under the protection of
	 * kpreempt_disable(), we allocate the ctxop outside the guard so its
	 * sleeping allocation will not cause a voluntary swtch().  This allows
	 * the rest of the initialization to proceed, ensuring valid state for
	 * the ctxop handlers.
	 */
	ctxop_attach(curthread, ctx);
	curthread->t_flag |= T_KFPU;

	if ((flags & KFPU_USE_LWP) == KFPU_USE_LWP) {
		/*
		 * For pure kernel threads with an LWP, we can use the LWP's
		 * pcb_fpu to save/restore context.
		 */
		fpu_ctx_t *pf = &pl->lwp_pcb.pcb_fpu;

		VERIFY(curthread->t_procp->p_flag & SSYS);
		VERIFY(kfpu == NULL);
		ASSERT((pf->fpu_flags & FPU_EN) == 0);

		/* Always restore the fpu to the initial state. */
		if (fp_save_mech == FP_XSAVE)
			pf->fpu_xsave_mask = XFEATURE_FP_ALL;
		fpinit();
		pf->fpu_flags = FPU_EN | FPU_KERNEL;
	} else {
		/* initialize the kfpu state */
		kernel_fpu_ctx_restore(kfpu);
	}
	kpreempt_enable();
}

void
kernel_fpu_end(kfpu_state_t *kfpu, uint_t flags)
{
	if ((curthread->t_flag & T_KFPU) == 0) {
		panic("curthread attempting to clear kernel FPU state "
		    "without using it");
	}

	/*
	 * General comments on why the rest of this function is structured the
	 * way it is. Be aware that there is a lot of subtlety here.
	 *
	 * If a user-level thread ever uses the fpu while in the kernel, then
	 * we cannot call fpdisable since that does STTS. That will set the
	 * ts bit in %cr0 which will cause an exception if anything touches the
	 * fpu. However, the user-level context switch handler (fpsave_ctxt)
	 * needs to access the fpu to save the registers into the pcb.
	 * fpsave_ctxt relies on CLTS having been done to clear the ts bit in
	 * fprestore_ctxt when the thread context switched onto the CPU.
	 *
	 * Calling fpdisable only effects the current CPU's %cr0 register.
	 *
	 * During ctxop_remove and kpreempt_enable, we can voluntarily context
	 * switch, so the CPU we were on when we entered this function might
	 * not be the same one we're on when we return from ctxop_remove or end
	 * the function. Note there can be user-level context switch handlers
	 * still installed if this is a user-level thread.
	 *
	 * We also must be careful in the unlikely chance we're running in an
	 * interrupt thread, since we can't leave the CPU's %cr0 TS state set
	 * incorrectly for the "real" thread to resume on this CPU.
	 */

	if ((flags & KFPU_NO_STATE) == 0) {
		kpreempt_disable();
	} else {
		ASSERT(curthread->t_preempt > 0);
	}

	curthread->t_flag &= ~T_KFPU;

	/*
	 * When we are ending things, we explicitly don't save the current
	 * kernel FPU state back to the temporary state. The kfpu API is not
	 * intended to be a permanent save location.
	 *
	 * If this is a user-level thread and we were to context switch
	 * before returning to user-land, fpsave_ctxt will be a no-op since we
	 * already saved the user-level FPU state the first time we run
	 * kernel_fpu_begin (i.e. we won't save the bad kernel fpu state over
	 * the user-level fpu state). The fpsave_ctxt functions only save if
	 * FPU_VALID is not already set. fp_save also set PCB_SET_UPDATE_FPU so
	 * fprestore_ctxt will be done in sys_rtt_common when the thread
	 * finally returns to user-land.
	 */

	if ((curthread->t_procp->p_flag & SSYS) != 0 &&
	    curthread->t_intr == NULL) {
		/*
		 * A kernel thread which is not an interrupt thread, so we
		 * STTS now.
		 */
		fpdisable();
	}

	if ((flags & KFPU_NO_STATE) == 0) {
		ctxop_remove(curthread, &kfpu_ctxop_tpl, kfpu);

		if (kfpu != NULL) {
			if (kfpu->kfpu_curthread != curthread) {
				panic("attempting to end kernel FPU state "
				    "for %p, but active thread is not "
				    "curthread", kfpu);
			} else {
				kfpu->kfpu_curthread = NULL;
			}
		}

		kpreempt_enable();
	}

	if (curthread->t_lwp != NULL) {
		uint_t f;

		if (flags & KFPU_USE_LWP) {
			f = FPU_EN | FPU_KERNEL;
		} else {
			f = FPU_KERNEL;
		}
		curthread->t_lwp->lwp_pcb.pcb_fpu.fpu_flags &= ~f;
	}
}

void
fpu_save_cache_init(void)
{
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpsave_cachep = kmem_cache_create("fxsave_cache",
		    sizeof (struct fxsave_state), FXSAVE_ALIGN,
		    NULL, NULL, NULL, NULL, NULL, 0);
		break;
	case FP_XSAVE:
		fpsave_cachep = kmem_cache_create("xsave_cache",
		    cpuid_get_xsave_size(), XSAVE_ALIGN,
		    NULL, NULL, NULL, NULL, NULL, 0);
		break;
	default:
		panic("Invalid fp_save_mech");
	}
}

/*
 * Fill in FPU information that is required by exec.
 */
void
fpu_auxv_info(int *typep, size_t *lenp)
{
	*typep = fp_elf;
	switch (fp_save_mech) {
	case FP_FXSAVE:
		*lenp = sizeof (struct fxsave_state);
		break;
	case FP_XSAVE:
		*lenp = cpuid_get_xsave_size();
		break;
	default:
		*lenp = 0;
		break;
	}
}

/*
 * This function exists to transform an xsave_state into an fxsave_state. The
 * way that we have to do this is nuanced. We assume that callers have already
 * handled FPU_EN and thus we only need to consider the xsave_state and its
 * component vector itself. This results in the following cases that we need to
 * consider:
 *
 *   o Neither the x87 / XMM state bits are set. We use the hardware default and
 *     need to ensure to copy the xsave header.
 *   o Both x87 / XMM state bits are set. We can copy everything.
 *   o Only the x87 bit is set. We need to copy the x87 state but make the XMM
 *     state be in the initial case.
 *   o Only the XMM bit is set. The reverse of the above case.
 *
 * The illumos and hardware defaults in 'sse_initial' and 'avx_initial' are
 * generally the same; however, the default floating point control word is
 * different.
 *
 * Finally, we have the complication of the MXCSR and MCXSR_MASK registers.
 * Because we are using xsave and xsaveopt in the kernel right now and not
 * xsavec, the hardware may write out the MXCSR and MXCSR_MASK registers if the
 * XFEATURE_AVX bit is set. Therefore if we don't have the XMM bit set but AVX
 * is set, we must also come back and copy out the MXCSR register. Sorry, we
 * don't make the rules.
 */
static void
fpu_xsave_to_fxsave(const struct xsave_state *xsave, struct fxsave_state *fx)
{
	const uint64_t comps = xsave->xs_header.xsh_xstate_bv;

	switch (comps & (XFEATURE_LEGACY_FP | XFEATURE_SSE)) {
	case XFEATURE_LEGACY_FP | XFEATURE_SSE:
		bcopy(xsave, fx, sizeof (*fx));
		return;
	case XFEATURE_LEGACY_FP:
		bcopy(xsave, fx, offsetof(struct fxsave_state, fx_xmm));
		fx->fx_mxcsr = SSE_MXCSR_INIT;
		fx->fx_mxcsr_mask = 0;
		break;
	case XFEATURE_SSE:
		bcopy(&sse_initial, fx, offsetof(struct fxsave_state,
		    fx_mxcsr));

		fx->fx_fcw = FPU_CW_INIT_HW;
		fx->fx_mxcsr = xsave->xs_fxsave.fx_mxcsr;
		fx->fx_mxcsr_mask = xsave->xs_fxsave.fx_mxcsr_mask;
		bcopy(xsave->xs_fxsave.fx_xmm, fx->fx_xmm, sizeof (fx->fx_xmm));
		break;
	default:
		bcopy(&sse_initial, fx, sizeof (*fx));
		fx->fx_fcw = FPU_CW_INIT_HW;
		break;
	}

	/*
	 * Account for the AVX causing MXCSR to be valid.
	 */
	if ((xsave->xs_header.xsh_xstate_bv & XFEATURE_AVX) != 0 &&
	    (xsave->xs_header.xsh_xstate_bv & XFEATURE_SSE) == 0) {
		fx->fx_mxcsr = xsave->xs_fxsave.fx_mxcsr;
		fx->fx_mxcsr_mask = xsave->xs_fxsave.fx_mxcsr_mask;
	}
}

/*
 * This function is designed to answer the question of are we using any xsave
 * family of instructions in context switch and therefore we have this state.
 * This should still remain true if we are using xsavec or xsaves in the kernel
 * in the future.
 */
boolean_t
fpu_xsave_enabled(void)
{
	return (fp_save_mech == FP_XSAVE);
}

/*
 * The following structure is used to track and manage the programmatic
 * construction of /proc and signal stack spilling of xsave information. All
 * known xsave types that the kernel supports must be included here.
 */
typedef struct xsave_proc_info {
	/*
	 * This matches the /proc xregs type that this data represents. This s
	 * used for /proc only.
	 */
	uint32_t xi_type;
	/*
	 * This indicates the size of the /proc data that we're operating on.
	 * This is only used for /proc.
	 */
	size_t	xi_size;
	/*
	 * This indicates the alignment that we want to have for the member when
	 * we're writing out. This is not used when setting data. This is only
	 * used for /proc.
	 */
	size_t	xi_align;
	/*
	 * This indicates whether this member must always be considered or not.
	 * This is used in both /proc and context/signal handling.
	 */
	bool	xi_always;
	/*
	 * This contains the corresponding bits in the xsave bit vector that
	 * corresponds to this entry. This is used for both /proc and
	 * context/signal handling.
	 */
	uint64_t xi_bits;
	/*
	 * The xi_fill function pointer is used to write out the /proc regset
	 * data (e.g. when a user reads xregs). This is only used for the /proc
	 * handling. The xi_valid function pointer is used instead to validate a
	 * given set of data that we've read in, while the xi_set pointer is
	 * used to actually transform the data in the underlying fpu save area.
	 */
	void	(*xi_fill)(const fpu_ctx_t *, const struct xsave_proc_info *,
	    void *);
	bool	(*xi_valid)(model_t, const void *);
	void	(*xi_set)(fpu_ctx_t *, const struct xsave_proc_info *,
	    uint64_t, const void *);
	/*
	 * The xi_signal_in and xi_signal_out function pointers are used for
	 * extended context and signal handling information. They are used when
	 * reading in data from a ucontext_t and writing it out respectively.
	 * These are only used for context/signal handling.
	 */
	int	(*xi_signal_in)(const struct xsave_proc_info *,
	    const ucontext_t *, const uc_xsave_t *, void *, uintptr_t *,
	    const uintptr_t);
	int	(*xi_signal_out)(const struct xsave_proc_info *, fpu_copyout_f,
	    uc_xsave_t *, const void *fpup, uintptr_t);
} xsave_proc_info_t;

static bool
fpu_proc_xregs_initial_state(const fpu_ctx_t *fpu, uint64_t feats)
{
	const struct xsave_state *xs = fpu->fpu_regs.kfpu_u.kfpu_xs;

	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == 0) {
		return (true);
	}

	return ((xs->xs_header.xsh_xstate_bv & feats) == 0);
}

static void
fpu_proc_xregs_xcr_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	prxregset_xcr_t *xcr = datap;

	xcr->prx_xcr_xcr0 = xsave_bv_all;
}

/*
 * Unlike other instruction portions, we treat the xsave header and the legacy
 * XMM section together as both are somewhat tied at the instruction hip. Unlike
 * the when dealing with other xsave regions like the ymm and zmm components,
 * the initial state here is much more nuanced as it has to match what we actual
 * do in the OS and depends on the components that are present.
 */
static void
fpu_proc_xregs_xsave_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	prxregset_xsave_t *prxsave = datap;
	const struct xsave_state *xsave = fpu->fpu_regs.kfpu_u.kfpu_xs;
	size_t hdr_off;

	/*
	 * In the x87/XMM case, the no device vs. initial state is different
	 * because the initial state case still wants us to copy the real xsave
	 * header. It's also worth calling out that the actual illumos default
	 * fxsave state is not the same as what Intel documents. The main
	 * difference is in what the x87 FPU control word is. This results in
	 * the following different cases that we need to think about:
	 *
	 *   o FPU_EN is not set. So we use the illumos default.
	 */
	if ((fpu->fpu_flags & FPU_EN) == 0) {
		bcopy(&avx_initial, prxsave, sizeof (*prxsave));
		return;
	}

	/*
	 * Convert all the fxsave region while taking into account the validity
	 * of the xsave bits. The prxregset_xsave_t structure is the same as the
	 * xsave structure in our ABI and Intel designed the xsave header to
	 * begin with the 512-bit fxsave structure.
	 */
	fpu_xsave_to_fxsave(xsave, (struct fxsave_state *)prxsave);

	/*
	 * Now that we've dealt with the x87 and XMM state, take care of the
	 * header.
	 */
	hdr_off = offsetof(prxregset_xsave_t, prx_xsh_xstate_bv);
	bcopy((const void *)((uintptr_t)xsave + hdr_off),
	    (void *)((uintptr_t)prxsave + hdr_off),
	    sizeof (struct xsave_header));
}

static void
fpu_proc_xregs_std_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	if (!fpu_proc_xregs_initial_state(fpu, info->xi_bits)) {
		size_t size, off;
		const void *xsave_off;

		cpuid_get_xsave_info(info->xi_bits, &size, &off);
		ASSERT3U(size, ==, info->xi_size);
		xsave_off = (void *)((uintptr_t)fpu->fpu_regs.kfpu_u.kfpu_xs +
		    off);
		bcopy(xsave_off, datap, info->xi_size);
	}
}

/*
 * Users are not allowed to actually set the xcr information this way. However,
 * to make it easier for someone to just do a read, modify, write, of the xregs
 * data, if it is identical, then we will accept it (and do nothing).
 */
static bool
fpu_proc_xregs_xcr_valid(model_t model, const void *datap)
{
	const prxregset_xcr_t *xcr = datap;

	return (xcr->prx_xcr_xcr0 == xsave_bv_all && xcr->prx_xcr_xfd == 0 &&
	    xcr->prx_xcr_pad[0] == 0 && xcr->prx_xcr_pad[1] == 0);
}

/*
 * To match traditional /proc semantics, we do not error if reserved bits of
 * MXCSR are set, they will be masked off when writing data. We do not allow
 * someone to indicate that they are asking for compressed xsave data, hence the
 * check that prx_xsh_comp_bv is zero. Separately, in fpu_proc_xregs_set() we
 * check that each component that was indicated in the xstate_bv is actually
 * present.
 */
static bool
fpu_proc_xregs_xsave_valid(model_t model, const void *datap)
{
	const prxregset_xsave_t *xsave = datap;
	uint64_t rsvd[6] = { 0 };

	if (bcmp(rsvd, xsave->prx_xsh_reserved, sizeof (rsvd)) != 0 ||
	    xsave->prx_xsh_xcomp_bv != 0) {
		return (false);
	}

	if ((xsave->prx_xsh_xstate_bv & ~xsave_bv_all) != 0) {
		return (false);
	}

	return (true);
}

/*
 * The YMM, ZMM, and Hi-ZMM registers are all valid when in an LP64 environment
 * on x86; however, when operating in ILP32, subsets are reserved. We require
 * that all reserved portions are set to zero.
 */
static bool
fpu_proc_xregs_ymm_valid(model_t model, const void *datap)
{
	upad128_t ymm_zero[8];
	const prxregset_ymm_t *ymm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&ymm_zero, sizeof (ymm_zero));
	return (bcmp(&ymm->prx_ymm[8], &ymm_zero, sizeof (ymm_zero)) == 0);
}

static bool
fpu_proc_xregs_zmm_valid(model_t model, const void *datap)
{
	upad256_t zmm_zero[8];
	const prxregset_zmm_t *zmm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&zmm_zero, sizeof (zmm_zero));
	return (bcmp(&zmm->prx_zmm[8], &zmm_zero, sizeof (zmm_zero)) == 0);
}

static bool
fpu_proc_xregs_hi_zmm_valid(model_t model, const void *datap)
{
	prxregset_hi_zmm_t hi_zmm_zero;
	const prxregset_hi_zmm_t *hi_zmm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&hi_zmm_zero, sizeof (hi_zmm_zero));
	return (bcmp(hi_zmm, &hi_zmm_zero, sizeof (hi_zmm_zero)) == 0);
}

/*
 * The xsave state consists of the first 512 bytes of the XMM state and then the
 * xsave header itself. Because of the xsave header, this structure is marked
 * with xi_always, so we must always process and consider it.
 *
 * Semantically if either of the bits around SSE / x87 is set, then we will copy
 * the entire thing. This may mean that we end up copying a region that is not
 * valid into the save area; however, that should be OK as we still have the
 * specific bit flags that indicate what we should consider or not.
 *
 * There is one additional wrinkle we need to consider and honor here. The CPU
 * will load the MXCSR values if the AVX bit is set in an xrstor regardless of
 * anything else. So if this is set and we do not have a valid x87/XMM bits
 * set then we will set the MXCSR to its default state in case the processor
 * tries to load it. For reference see:
 *
 *   o Intel SDM Volume 1: 13.8.1 Standard Form of XRSTOR
 *   o AMD64 Volume 2: Section 11.5.9 MXCSR State Management
 *
 * Note, the behavior around this changes depending on whether using the
 * compressed xrstor or not. We are not, but it's worth being aware of. We do
 * not worry about MXCSR_MASK because the instructions ignore it.
 */
static void
fpu_proc_xregs_xsave_set(fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    uint64_t xsave_bv, const void *datap)
{
	const struct xsave_state *src_xs = datap;
	struct xsave_state *targ_xs = fpu->fpu_regs.kfpu_u.kfpu_xs;

	if ((xsave_bv & info->xi_bits) != 0) {
		bcopy(&src_xs->xs_fxsave, &targ_xs->xs_fxsave,
		    sizeof (struct fxsave_state));
	} else if ((xsave_bv & XFEATURE_AVX) != 0) {
		targ_xs->xs_fxsave.fx_mxcsr = SSE_MXCSR_INIT;
	}

	bcopy(&src_xs->xs_header, &targ_xs->xs_header,
	    sizeof (struct xsave_header));
	targ_xs->xs_fxsave.fx_mxcsr &= sse_mxcsr_mask;
}

static void
fpu_proc_xregs_std_set(fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    uint64_t xsave_bv, const void *datap)
{
	size_t size, off;
	void *xsave_off;

	cpuid_get_xsave_info(info->xi_bits, &size, &off);
	xsave_off = (void *)((uintptr_t)fpu->fpu_regs.kfpu_u.kfpu_xs +
	    off);
	bcopy(datap, xsave_off, size);
}

/*
 * Dealing with XMM data is a little more annoying in signal context. If UC_FPU
 * is set, the ucontext_t's fpregset_t contains a copy of the XMM region. That
 * must take priority over an XMM region that showed up in the uc_xsave_t data.
 * In the signal copyout code we do not save XMM region in the uc_xsave_t or set
 * it as a present component because of it being kept in the fpregset_t. Because
 * of this behavior, if we find the XMM (or x87) state bits present, we treat
 * that as an error.
 *
 * The system has always gone through and cleaned up the reserved bits in the
 * fxsave state when someone calls setcontext(). Therefore we need to do the
 * same thing which is why you see the masking of the mxcsr below.
 *
 * Finally, there is one last wrinkle here that we need to consider. The
 * fpregset_t has two private words which cache the status/exception
 * information. Therefore, we well...  cheat. Intel has left bytes 464 (0x1d0)
 * through 511 (0x1ff) available for us to do what we want. So we will pass this
 * through that for the moment to help us pass this state around without too
 * much extra allocation.
 */
static int
fpu_signal_copyin_xmm(const xsave_proc_info_t *info, const ucontext_t *kuc,
    const uc_xsave_t *ucx, void *fpup, uintptr_t *udatap,
    const uintptr_t max_udata)
{
	struct xsave_state *xsave = fpup;

	if ((ucx->ucx_bv & info->xi_bits) != 0) {
		return (EINVAL);
	}

	if ((kuc->uc_flags & UC_FPU) != 0) {
		bcopy(&kuc->uc_mcontext.fpregs, &xsave->xs_fxsave,
		    sizeof (struct fxsave_state));
		xsave->xs_fxsave.__fx_ign2[3]._l[0] =
		    kuc->uc_mcontext.fpregs.fp_reg_set.fpchip_state.status;
		xsave->xs_fxsave.__fx_ign2[3]._l[1] =
		    kuc->uc_mcontext.fpregs.fp_reg_set.fpchip_state.xstatus;
		xsave->xs_fxsave.fx_mxcsr &= sse_mxcsr_mask;
		xsave->xs_header.xsh_xstate_bv |= info->xi_bits;
	}

	return (0);
}

static int
fpu_signal_copyin_std(const xsave_proc_info_t *info, const ucontext_t *kuc,
    const uc_xsave_t *ucx, void *fpup, uintptr_t *udatap,
    const uintptr_t max_udata)
{
	size_t len, xsave_off;
	void *copy_to;
	struct xsave_state *xsave = fpup;

	cpuid_get_xsave_info(info->xi_bits, &len, &xsave_off);
	if (*udatap + len > max_udata) {
		return (EOVERFLOW);
	}

	copy_to = (void *)((uintptr_t)fpup + xsave_off);
	if (ddi_copyin((void *)*udatap, copy_to, len, 0) != 0) {
		return (EFAULT);
	}

	xsave->xs_header.xsh_xstate_bv |= info->xi_bits;
	*udatap = *udatap + len;

	return (0);
}

static int
fpu_signal_copyout_std(const xsave_proc_info_t *info, fpu_copyout_f copyfunc,
    uc_xsave_t *ucx, const void *fpup, uintptr_t udatap)
{
	size_t len, xsave_off;
	const void *copy_from;
	void *copy_to;
	int ret;

	cpuid_get_xsave_info(info->xi_bits, &len, &xsave_off);
	copy_from = (void *)(uintptr_t)fpup + xsave_off;
	copy_to = (void *)(udatap + ucx->ucx_len);

	ret = copyfunc(copy_from, copy_to, len);
	if (ret != 0) {
		return (ret);
	}

	ucx->ucx_len += len;
	ucx->ucx_bv |= info->xi_bits;
	return (0);
}

/*
 * This table contains information about the extended FPU states and synthetic
 * information we create for /proc, the ucontext_t, and signal handling. The
 * definition of the xsave_proc_info_t describes how each member is used.
 *
 * In general, this table is expected to be in the order of the xsave data
 * structure itself. Synthetic elements that we create can go anywhere and new
 * ones should be inserted at the end. This structure is walked in order to
 * produce the /proc and signal handling logic, so changing the order is
 * meaningful for those and should not be done lightly.
 */
static const xsave_proc_info_t fpu_xsave_info[] = { {
	.xi_type = PRX_INFO_XCR,
	.xi_size = sizeof (prxregset_xcr_t),
	.xi_align = alignof (prxregset_xcr_t),
	.xi_always = true,
	.xi_bits = 0,
	.xi_fill = fpu_proc_xregs_xcr_fill,
	.xi_valid = fpu_proc_xregs_xcr_valid
}, {
	/*
	 * The XSAVE entry covers both the xsave header and the %xmm registers.
	 * Note, there is no signal copyout information for the %xmm registers
	 * because it is expected that that data is already in the fpregset_t.
	 */
	.xi_type = PRX_INFO_XSAVE,
	.xi_size = sizeof (prxregset_xsave_t),
	.xi_align = FPU_ALIGN_XMM,
	.xi_always = true,
	.xi_bits = XFEATURE_LEGACY_FP | XFEATURE_SSE,
	.xi_fill = fpu_proc_xregs_xsave_fill,
	.xi_set = fpu_proc_xregs_xsave_set,
	.xi_valid = fpu_proc_xregs_xsave_valid,
	.xi_signal_in = fpu_signal_copyin_xmm
}, {
	.xi_type = PRX_INFO_YMM,
	.xi_size = sizeof (prxregset_ymm_t),
	.xi_align = FPU_ALIGN_YMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_valid = fpu_proc_xregs_ymm_valid,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	/*
	 * There is no /proc validation function for the mask registers because
	 * they are the same in ILP32 / LP64 and there is nothing for us to
	 * actually validate.
	 */
	.xi_type = PRX_INFO_OPMASK,
	.xi_size = sizeof (prxregset_opmask_t),
	.xi_align = alignof (prxregset_opmask_t),
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_OPMASK,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	.xi_type = PRX_INFO_ZMM,
	.xi_size = sizeof (prxregset_zmm_t),
	.xi_align = FPU_ALIGN_ZMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_ZMM,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_valid = fpu_proc_xregs_zmm_valid,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	.xi_type = PRX_INFO_HI_ZMM,
	.xi_size = sizeof (prxregset_hi_zmm_t),
	.xi_align = FPU_ALIGN_ZMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_HI_ZMM,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_valid = fpu_proc_xregs_hi_zmm_valid,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
} };

static bool
fpu_proc_xregs_include(const xsave_proc_info_t *infop)
{
	return (infop->xi_always || (xsave_bv_all & infop->xi_bits) != 0);
}

void
fpu_proc_xregs_info(struct proc *p __unused, uint32_t *ninfop, uint32_t *sizep,
    uint32_t *dstart)
{
	size_t ret = sizeof (prxregset_hdr_t);
	uint32_t ninfo = 0;

	ASSERT(fpu_xsave_enabled());

	/*
	 * Right now the set of flags that are enabled in the FPU is global.
	 * That is, while the pcb's fcpu_ctx_t has the fpu_xsave_mask, the
	 * actual things that might show up and we care about are all about what
	 * is set up in %xcr0 which is stored in the global xsave_bv_all. If we
	 * move to per-process FPU enablement which is likely to come with AMX,
	 * then this will need the proc_t to look at, hence why we've set things
	 * up with the unused variable above.
	 *
	 * We take two passes through the array. The first is just to count up
	 * how many informational entries we need.
	 */
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;
		ninfo++;
	}

	ASSERT3U(ninfo, >, 0);
	ret += sizeof (prxregset_info_t) * ninfo;

	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		size_t curphase;
		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;

		curphase = ret % fpu_xsave_info[i].xi_align;
		if (ret < fpu_xsave_info[i].xi_align) {
			ret = fpu_xsave_info[i].xi_align;
		} else if (curphase != 0) {
			ret += curphase;
		}

		if (i == 0 && dstart != NULL) {
			*dstart = ret;
		}

		ret += fpu_xsave_info[i].xi_size;
	}

	VERIFY3U(ret, <=, UINT32_MAX);
	if (sizep != NULL) {
		*sizep = ret;
	}

	if (ninfop != NULL) {
		*ninfop = ninfo;
	}
}

/*
 * This function supports /proc. Because /proc does not have a process locked
 * while processing a PCSXREG, this tries to establish an upper bound that we
 * will validate later in fpu_proc_xregs_set(). We basically say that if you
 * take the maximum xsave size and add 1 KiB that is a good enough approximation
 * for the maximum size. The 1 KiB is us basically trying to rationalize the
 * overhead of our structures that we're adding right, while being cognisant of
 * differing alignments and the fact that the full xsave size is in some cases
 * (when supervisor states or features we don't support are present) going to be
 * larger than we would need for this.
 */
size_t
fpu_proc_xregs_max_size(void)
{
	VERIFY(fpu_xsave_enabled());
	return (cpuid_get_xsave_size() + 0x1000);
}

/*
 * This functions supports /proc. In particular, it's meant to perform the
 * following:
 *
 *  o Potentially save the current thread's registers.
 *  o Write out the x86 xsave /proc xregs format data from the xsave data we
 *    actually have. Note, this can be a little weird for cases where the FPU is
 *    not actually enabled, which happens for system processes.
 */
void
fpu_proc_xregs_get(klwp_t *lwp, void *buf)
{
	uint32_t size, ninfo, curinfo, dstart;
	fpu_ctx_t *fpu = &lwp->lwp_pcb.pcb_fpu;
	prxregset_hdr_t *hdr = buf;

	ASSERT(fpu_xsave_enabled());
	fpu_proc_xregs_info(lwp->lwp_procp, &ninfo, &size, &dstart);

	/*
	 * Before we get going, defensively zero out all the data buffer so that
	 * the rest of the fill functions can assume a specific base.
	 */
	bzero(buf, size);

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * This case suggests that thread in question doesn't have a
		 * valid FPU save state which should only happen when it is on
		 * CPU. If this is the case, we must ensure that we save the
		 * current FPU state before proceeding. We also sanity check
		 * several things here before doing this as using /proc on
		 * yourself is always exciting. fp_save() will ensure that the
		 * thread is flagged to go back to being an eager FPU before
		 * returning back to userland.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_save(fpu);
	}
	kpreempt_enable();

	hdr->pr_type = PR_TYPE_XSAVE;
	hdr->pr_size = size;
	hdr->pr_flags = hdr->pr_pad[0] = hdr->pr_pad[1] = hdr->pr_pad[2] =
	    hdr->pr_pad[3] = 0;
	hdr->pr_ninfo = ninfo;

	curinfo = 0;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		void *startp;
		uint32_t phase;

		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;

		phase = dstart % fpu_xsave_info[i].xi_align;
		if (dstart < fpu_xsave_info[i].xi_align) {
			ASSERT3U(i, !=, 0);
			dstart = fpu_xsave_info[i].xi_align;
		} else if (phase != 0) {
			ASSERT3U(i, !=, 0);
			dstart += phase;
		}

		hdr->pr_info[curinfo].pri_type = fpu_xsave_info[i].xi_type;
		hdr->pr_info[curinfo].pri_flags = 0;
		hdr->pr_info[curinfo].pri_size = fpu_xsave_info[i].xi_size;
		hdr->pr_info[curinfo].pri_offset = dstart;

		startp = (void *)((uintptr_t)buf + dstart);
		fpu_xsave_info[i].xi_fill(fpu, &fpu_xsave_info[i], startp);
		dstart += fpu_xsave_info[i].xi_size;
		ASSERT3U(curinfo, <=, ninfo);
		curinfo++;
	}
}

/*
 * We have been asked to set the data in the FPU for a given thread. Our
 * prmachdep code has already validated that the raw semantics of the data that
 * we have are valid (that is the appropriate sizes, offsets, and flags). We now
 * apply additional checking here:
 *
 *   o The xsave structure is present and only valid bits are set.
 *   o If the xsave component bit-vector is set, we have the corresponding proc
 *     info item.
 *   o Read-only items are ignored if and only if they actually match what we
 *     gave the user mostly as a courtesy to simplify things here.
 *   o ILP32 processes which can't support many of the regions are allowed to
 *     have the items here (as we likely gave them to them), but they must be
 *     zero if they are set.
 *
 * We take a first pass through all the data, validating it makes sense for the
 * FPU. Only after that point do we ensure that we have the FPU data in question
 * and then we clobber all the FPU data. Part of the semantics of setting this
 * is that we're setting the entire extended FPU.
 */
int
fpu_proc_xregs_set(klwp_t *lwp, void *buf)
{
	prxregset_hdr_t *prx = buf;
	model_t model = lwp_getdatamodel(lwp);
	uint64_t bv_found = 0;
	const prxregset_xsave_t *xsave = NULL;
	fpu_ctx_t *fpu = &lwp->lwp_pcb.pcb_fpu;

	VERIFY(fpu_xsave_enabled());

	/*
	 * First, walk each note info header that we have from the user and
	 * proceed to validate it. The prmachdep code has already validated that
	 * the size, type, and offset information is valid, but it has not
	 * validated the semantic contents of this or if someone is trying to
	 * write something they shouldn't.
	 *
	 * While we walk this, we keep track of where the xsave header is. We
	 * also track all of the bits that we have found along the way so we can
	 * match up and ensure that everything that was set has a corresponding
	 * bit in the xsave bitmap. If we have something in the xsave bitmap,
	 * but not its corresponding data, then that is an error. However, we
	 * allow folks to write data regions without the bit set in the xsave
	 * data to make the read, modify, write process simpler.
	 */
	for (uint32_t i = 0; i < prx->pr_ninfo; i++) {
		const prxregset_info_t *info = &prx->pr_info[i];
		bool found = false;

		for (size_t pt = 0; pt < ARRAY_SIZE(fpu_xsave_info); pt++) {
			void *data;
			if (info->pri_type != fpu_xsave_info[pt].xi_type)
				continue;

			found = true;
			data = (void *)((uintptr_t)buf + info->pri_offset);
			if (fpu_xsave_info[pt].xi_valid != NULL &&
			    !fpu_xsave_info[pt].xi_valid(model, data)) {
				return (EINVAL);
			}

			if (info->pri_type == PRX_INFO_XSAVE) {
				xsave = data;
			}
			bv_found |= fpu_xsave_info[pt].xi_bits;
			break;
		}

		if (!found) {
			return (EINVAL);
		}
	}

	/*
	 * No xsave data, no dice.
	 */
	if (xsave == NULL) {
		return (EINVAL);
	}

	/*
	 * If anything is set in the xsave header that was not found as we
	 * walked structures, then that is an error. The opposite is not true as
	 * discussed above.
	 */
	if ((xsave->prx_xsh_xstate_bv & ~bv_found) != 0) {
		return (EINVAL);
	}

	/*
	 * At this point, we consider all the data actually valid. Now we must
	 * set up this information in the save area. If this is our own lwp, we
	 * must disable it first. Otherwise, we expect that it is already valid.
	 * To try to sanitize this, we will defensively zero the entire region
	 * as we are setting everything that will result in here.
	 */
	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * This case suggests that thread in question doesn't have a
		 * valid FPU save state which should only happen when it is on
		 * CPU. If this is the case, we explicitly disable the FPU, but
		 * do not save it before proceeding. We also sanity check
		 * several things here before doing this as using /proc on
		 * yourself is always exciting. Unlike fp_save(), fp_free() does
		 * not signal that an update is required, so we unconditionally
		 * set that for all threads.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_free(fpu);
	}
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	bzero(lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
	    cpuid_get_xsave_size());

	for (uint32_t i = 0; i < prx->pr_ninfo; i++) {
		const prxregset_info_t *info = &prx->pr_info[i];
		bool found = false;

		for (size_t pt = 0; pt < ARRAY_SIZE(fpu_xsave_info); pt++) {
			const void *data;
			if (info->pri_type != fpu_xsave_info[pt].xi_type)
				continue;

			/*
			 * Check if we have a set function and if we should
			 * include this. We may not if this is something like
			 * PRX_INFO_XCR which is read-only.
			 *
			 * We may not include a given entry as it may not have
			 * been set in the actual xsave state that we have been
			 * asked to restore, in which case to not break the
			 * xsaveopt logic, we must leave it in its initial
			 * state, e.g. zeroed (generally). XMM data initial
			 * state is not zeroed, but is marked with xi_always to
			 * help account for this.
			 */
			found = true;
			if (fpu_xsave_info[pt].xi_set == NULL)
				break;
			if (!fpu_xsave_info[pt].xi_always &&
			    (xsave->prx_xsh_xstate_bv &
			    fpu_xsave_info[pt].xi_bits) !=
			    fpu_xsave_info[pt].xi_bits) {
				break;
			}

			data = (void *)((uintptr_t)buf + info->pri_offset);
			fpu_xsave_info[pt].xi_set(fpu, &fpu_xsave_info[pt],
			    xsave->prx_xsh_xstate_bv, data);
		}

		VERIFY(found);
	}
	kpreempt_enable();

	return (0);
}

/*
 * To be included in the signal copyout logic we must have a copy function and
 * the bit in question must be included. Note, we don't consult xi_always here
 * as that is really part of what is always present for xsave logic and
 * therefore isn't really pertinent here because of our custom format. See the
 * big theory statement for more info.
 */
static bool
fpu_signal_include(const xsave_proc_info_t *infop, uint64_t xs_bv)
{
	return ((infop->xi_bits & xs_bv) == infop->xi_bits &&
	    infop->xi_signal_out != NULL);
}

/*
 * We need to fill out the xsave related data into the ucontext_t that we've
 * been given. We should have a valid user pointer at this point in the uc_xsave
 * member. This is much simpler than the copyin that we have. Here are the
 * current assumptions:
 *
 *   o This is being called for the current thread. This is not meant to operate
 *     on an arbitrary thread's state.
 *   o We cannot assume whether the FPU is valid in the pcb or not. While most
 *     callers will have just called getfpregs() which saved the state, don't
 *     assume that.
 *   o We assume that the user address has the requisite required space for this
 *     to be copied out.
 *   o We assume that copyfunc() will ensure we are not copying into a kernel
 *     address.
 *
 * For more information on the format of the data, see the 'Signal Handling and
 * the ucontext_t' portion of the big theory statement. We copy out all the
 * constituent parts and then come back and write out the actual final header
 * information.
 */
int
fpu_signal_copyout(klwp_t *lwp, uintptr_t uaddr, fpu_copyout_f copyfunc)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	uint64_t xs_bv;
	uc_xsave_t ucx;
	int ret;

	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	VERIFY3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	if (!fpu_xsave_enabled()) {
		return (ENOTSUP);
	}

	/*
	 * Unlike when we're dealing with /proc, we can unconditionally call
	 * fp_save() because this is always called in the context where the lwp
	 * we're operating on is always the one on CPU (which is what fp_save()
	 * asserts).
	 */
	fp_save(fpu);

	bzero(&ucx, sizeof (ucx));
	ucx.ucx_vers = UC_XSAVE_VERS;
	ucx.ucx_len += sizeof (uc_xsave_t);

	xs_bv = fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		const xsave_proc_info_t *info = &fpu_xsave_info[i];

		if (!fpu_signal_include(&fpu_xsave_info[i], xs_bv))
			continue;
		ret = info->xi_signal_out(info, copyfunc, &ucx,
		    lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
		    uaddr);
		if (ret != 0) {
			kpreempt_enable();
			return (ret);
		}
	}

	/*
	 * Now that everything has been copied out, we should have an accurate
	 * value in the uc_xsave_t header and we can copy that out at the start
	 * of the user data.
	 */
	ret = copyfunc(&ucx, (void *)uaddr, sizeof (ucx));
	return (ret);
}

/*
 * Here we've been given a ucontext_t which potentially has a user pointer to
 * xsave state that we've copied out previously. In this case we need to do the
 * following, assuming UC_XSAVE is present:
 *
 *   o Copy in our header and validate it.
 *   o Allocate an fpu context to use as a holding ground for all this data.
 *   o If UC_FPU is set, override the xsave structure with the saved XMM state,
 *     clear UC_FPU, and make sure that the correct xsave_bv bits are set.
 *
 * Currently we always allocate the additional state as a holding ground for the
 * FPU. What we're copying in may not be valid and we don't want to clobber the
 * existing FPU state or deal with merging it until we believe it's reasonable
 * enough. The proc_t is here to set us up for when we have per-process settings
 * in the extended feature disable MSRs.
 */
int
fpu_signal_copyin(klwp_t *lwp, ucontext_t *kuc)
{
	uc_xsave_t ucx;
	uint64_t bv;
	uintptr_t data, max_data;
	void *fpu;
	proc_t *p = lwp->lwp_procp;
	size_t ksize;

	/*
	 * Because this has been opaque filler and the kernel has never
	 * historically looked at it, we don't really care about the uc_xsave
	 * pointer being garbage in the case that the flag is not set. While
	 * this isn't perhaps the most sporting choice in some cases, this is on
	 * the other hand, pragmatic.
	 */
	if ((kuc->uc_flags & UC_XSAVE) != 0) {
		if (kuc->uc_xsave == 0) {
			return (EINVAL);
		}

		if (!fpu_xsave_enabled()) {
			return (ENOTSUP);
		}
	} else {
		return (0);
	}

	if (ddi_copyin((const void *)kuc->uc_xsave, &ucx, sizeof (ucx), 0) !=
	    0) {
		return (EFAULT);
	}

	ksize = cpuid_get_xsave_size();
	if (ucx.ucx_vers != UC_XSAVE_VERS || ucx.ucx_len < sizeof (ucx) ||
	    ucx.ucx_len > ksize ||
	    (ucx.ucx_bv & ~xsave_bv_all) != 0 ||
	    (uintptr_t)p->p_as->a_userlimit - ucx.ucx_len <
	    (uintptr_t)kuc->uc_xsave) {
		return (EINVAL);
	}

	/*
	 * OK, our goal right now is to recreate a valid xsave_state structure
	 * that we'll ultimately end up having to merge with our existing one in
	 * the FPU save state. The reason we describe this as a merge is to help
	 * future us when we want to retain supervisor state which will never be
	 * part of userland signal state. The design of the userland signal
	 * state is basically to compress it as much as we can. This is done for
	 * two reasons:
	 *
	 *   1) We currently consider this a private interface.
	 *   2) We really want to minimize the actual amount of stack space we
	 *	use as much as possible. Most applications aren't using AVX-512
	 *	right now, so doing our own compression style is worthwhile. If
	 *	libc adopts AVX-512 routines, we may want to change this.
	 *
	 * On the allocation below, our assumption is that if a thread has taken
	 * a signal, then it is likely to take a signal again in the future (or
	 * be shortly headed to its demise). As such, when that happens we will
	 * leave the allocated signal stack around for the process. Most
	 * applications don't allow all threads to take signals, so this should
	 * hopefully help amortize the cost of the allocation.
	 */
	max_data = (uintptr_t)kuc->uc_xsave + ucx.ucx_len;
	data = (uintptr_t)kuc->uc_xsave + sizeof (ucx);
	bv = ucx.ucx_bv;
	if (lwp->lwp_pcb.pcb_fpu.fpu_signal == NULL) {
		lwp->lwp_pcb.pcb_fpu.fpu_signal =
		    kmem_cache_alloc(fpsave_cachep, KM_SLEEP);
	}
	fpu = lwp->lwp_pcb.pcb_fpu.fpu_signal;

	/*
	 * Unconditionally initialize the memory we get in here to ensure that
	 * it is in a reasonable state for ourselves. This ensures that unused
	 * regions are mostly left in their initial state (the main exception
	 * here is the x87/XMM state, but that should be OK). We don't fill in
	 * the initial xsave state as we expect that to happen as part of our
	 * processing.
	 */
	bzero(fpu, ksize);

	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		int ret;
		const xsave_proc_info_t *info = &fpu_xsave_info[i];
		if (!info->xi_always && (info->xi_bits & bv) == 0)
			continue;
		bv &= ~info->xi_bits;

		if (info->xi_signal_in == NULL)
			continue;
		ret = info->xi_signal_in(info, kuc, &ucx, fpu, &data, max_data);
		if (ret != 0) {
			return (ret);
		}
	}
	ASSERT0(bv);

	/*
	 * As described in the big theory statement section 'Signal Handling and
	 * the ucontext_t', we always remove UC_FPU from here as we've taken
	 * care of reassembling it ourselves.
	 */
	kuc->uc_flags &= ~UC_FPU;
	kuc->uc_xsave = (uintptr_t)fpu;

	return (0);
}

/*
 * This determines the size of the signal stack that we need for our custom form
 * of the xsave state.
 */
size_t
fpu_signal_size(klwp_t *lwp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	size_t len = sizeof (uc_xsave_t);
	uint64_t xs_bv;

	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	VERIFY3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	if (!fpu_xsave_enabled()) {
		return (0);
	}

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		fp_save(fpu);
	}

	xs_bv = fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		size_t comp_size;

		if (!fpu_signal_include(&fpu_xsave_info[i], xs_bv))
			continue;

		cpuid_get_xsave_info(fpu_xsave_info[i].xi_bits, &comp_size,
		    NULL);
		len += comp_size;
	}

	kpreempt_enable();
	return (len);
}

/*
 * This function is used in service of restorecontext() to set the specified
 * thread's extended FPU state to the passed in data. Our assumptions at this
 * point from the system are:
 *
 *   o Someone has already verified that the actual xsave header is correct.
 *   o Any traditional XMM state that causes a #gp has been clamped.
 *   o That data is basically the correct sized xsave state structure. Right now
 *     that means it is not compressed and follows the CPUID-based rules for
 *     constructing and laying out data.
 *   o That the lwp argument refers to the current thread.
 *
 * Our primary purpose here is to merge the current FPU state with what exists
 * here. Right now, "merge", strictly speaking is just "replace". We can get
 * away with just replacing everything because all we currently save are user
 * states. If we start saving kernel states in here, this will get more nuanced
 * and we will need to be more careful about how we store data here.
 */
void
fpu_set_xsave(klwp_t *lwp, const void *data)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	uint32_t status, xstatus;
	struct xsave_state *dst_xsave;

	VERIFY(fpu_xsave_enabled());
	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	ASSERT3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	/*
	 * We use fp_save() here rather than a stock fpdisable() so we can
	 * attempt to honor our invariants that when the thread state has been
	 * saved, the valid flag is set, even though we're going to be
	 * overwriting it shortly. If we just called fpdisable() then we would
	 * basically be asking for trouble.
	 *
	 * Because we are modifying the state here and we don't want the system
	 * to end up in an odd state, we are being a little paranoid and
	 * disabling preemption across this operation. In particular, once the
	 * state is properly tagged with FPU_VALID, there should be no other way
	 * that this thread can return to userland and get cleared out because
	 * we're resetting its context; however, we let paranoia win out.
	 */
	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		fp_save(fpu);
	}

	bcopy(data, lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
	    cpuid_get_xsave_size());
	dst_xsave = lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic;
	status = dst_xsave->xs_fxsave.__fx_ign2[3]._l[0];
	xstatus = dst_xsave->xs_fxsave.__fx_ign2[3]._l[1];
	dst_xsave->xs_fxsave.__fx_ign2[3]._l[0] = 0;
	dst_xsave->xs_fxsave.__fx_ign2[3]._l[1] = 0;

	/*
	 * These two status words are information that the kernel itself uses to
	 * track additional information and is part of the traditional fpregset,
	 * but is not part of our xregs information. Because we are setting this
	 * state, we leave it up to the rest of the kernel to determine whether
	 * this came from an fpregset_t or is being reset to the default of 0.
	 */
	fpu->fpu_regs.kfpu_status = status;
	fpu->fpu_regs.kfpu_xstatus = xstatus;

	fpu->fpu_flags |= FPU_VALID;
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	kpreempt_enable();
}

/*
 * Convert the current FPU state to the traditional fpregset_t. In the 64-bit
 * kernel, this is just an fxsave_state with additional values for the status
 * and xstatus members.
 *
 * This has the same nuance as the xregs cases discussed above, but is simpler
 * in that we only need to handle the fxsave state, but more complicated because
 * we need to check our save mechanism.
 */
void
fpu_get_fpregset(klwp_t *lwp, fpregset_t *fp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;

	kpreempt_disable();
	fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
	fp->fp_reg_set.fpchip_state.xstatus = fpu->fpu_regs.kfpu_xstatus;

	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * If we're requesting the fpregs of a thread that isn't
		 * currently valid and isn't the one that we're executing, then
		 * we consider getting this information to be a best-effort and
		 * we will not stop the thread in question to serialize it,
		 * which means possibly getting stale data. This is the
		 * traditional semantics that the system has used to service
		 * this for /proc.
		 */
		if (curthread == lwptot(lwp)) {
			VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
			fp_save(fpu);
		}
	}

	/*
	 * If the FPU is not enabled and the state isn't valid (due to someone
	 * else setting it), just copy the initial state.
	 */
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == 0) {
		bcopy(&sse_initial, fp, sizeof (sse_initial));
		kpreempt_enable();
		return;
	}

	/*
	 * Given that we have an enabled FPU, we must look at the type of FPU
	 * save mechanism to clean this up. In particular, while we can just
	 * copy the save area with FXSAVE, with XSAVE we must carefully copy
	 * only the bits that are valid and reset the rest to their default
	 * state.
	 */
	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(fpu->fpu_regs.kfpu_u.kfpu_fx, fp,
		    sizeof (struct fxsave_state));
		break;
	case FP_XSAVE:
		fpu_xsave_to_fxsave(fpu->fpu_regs.kfpu_u.kfpu_xs,
		    (struct fxsave_state *)fp);
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	kpreempt_enable();
}

/*
 * This is a request to set the ABI fpregset_t into our actual hardware state.
 * In the 64-bit kernel the first 512 bytes of the fpregset_t is the same as the
 * 512-byte fxsave area.
 */
void
fpu_set_fpregset(klwp_t *lwp, const fpregset_t *fp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * We always save the entire FPU. This is required if we're
		 * using xsave. If we're using fxsave, we could skip the
		 * 512-byte write and instead just disable the FPU since we'd be
		 * replacing it all. For now we don't bother with more
		 * conditional logic.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_save(fpu);
	}

	fpu->fpu_regs.kfpu_xstatus = fp->fp_reg_set.fpchip_state.xstatus;
	fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_fx,
		    sizeof (struct fxsave_state));
		break;
	case FP_XSAVE:
		bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_xs,
		    sizeof (struct fxsave_state));
		fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP | XFEATURE_SSE;
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	fpu->fpu_flags |= FPU_VALID;
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	kpreempt_enable();
}