60  * (a) for each task, we need to remember which CPU was the last one to have
61  *     the task's FPSIMD state loaded into its FPSIMD registers;
62  * (b) for each CPU, we need to remember which task's userland FPSIMD state has
69  * address of the userland FPSIMD state of the task that was loaded onto the CPU
75  * task's fpsimd_cpu are still mutually in sync. If this is the case, we
79  * indicate whether or not the userland FPSIMD state of the current task is
82  * task. If the task is behaving as a VMM, then this is will be managed by
89  * called from softirq context, which will save the task's FPSIMD context back
91  * task's FPSIMD state from task context and thereby corrupting the state, it
92  * is necessary to protect any manipulation of a task's fpsimd_state or
96  * For a certain task, the sequence may look something like this:
97  * - the task gets scheduled in; if both the task's fpsimd_cpu field
99  *   variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
102  * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
103  *   userland FPSIMD state is copied from memory to the registers, the task's
105  *   CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
108  * - the task executes an ordinary syscall; upon return to userland, the
112  * - the task executes a syscall which executes some NEON instructions; this is
113  *   preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
117  * - the task gets preempted after kernel_neon_end() is called; as we have not
262 unsigned int task_get_vl(const struct task_struct *task, enum vec_type type)  in task_get_vl()  argument
264 	return task->thread.vl[type];  in task_get_vl()
267 void task_set_vl(struct task_struct *task, enum vec_type type,  in task_set_vl()  argument
270 	task->thread.vl[type] = vl;  in task_set_vl()
273 unsigned int task_get_vl_onexec(const struct task_struct *task,  in task_get_vl_onexec()  argument
276 	return task->thread.vl_onexec[type];  in task_get_vl_onexec()
279 void task_set_vl_onexec(struct task_struct *task, enum vec_type type,  in task_set_vl_onexec()  argument
282 	task->thread.vl_onexec[type] = vl;  in task_set_vl_onexec()
286  * TIF_SME controls whether a task can use SME without trapping while
297  * TIF_SVE controls whether a task can use SVE without trapping while
298  * in userspace, and also (together with TIF_SME) the way a task's
301  * The kernel uses this flag to track whether a user task is actively
308  *    The task can execute SVE instructions while in userspace without
316  *    An attempt by the user task to execute an SVE instruction causes
327  *    When the FPSIMD only state stored task->thread.fp_type is set to
329  *    task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
335  *    task->thread.sve_state does not need to be non-NULL, valid or any
341  *    When the full SVE state is stored task->thread.fp_type is set to
344  *    task->thread.sve_state, formatted appropriately for vector
345  *    length task->thread.sve_vl or, if SVCR.SM is set,
346  *    task->thread.sme_vl. The storage for the vector registers in
347  *    task->thread.uw.fpsimd_state should be ignored.
349  *    task->thread.sve_state must point to a valid buffer at least
350  *    sve_state_size(task) bytes in size. The data stored in
351  *    task->thread.uw.fpsimd_state.vregs should be considered stale
354  *  * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
378 			/* Stop tracking SVE for this task until next use. */  in task_fpsimd_load()
657  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
658  * task->thread.sve_state.
660  * Task can be a non-runnable task, or current.  In the latter case,
663  * task->thread.sve_state must point to at least sve_state_size(task)
665  * task->thread.uw.fpsimd_state must be up to date before calling this
668 static inline void fpsimd_to_sve(struct task_struct *task)  in fpsimd_to_sve()  argument
671 	void *sst = task->thread.sve_state;  in fpsimd_to_sve()
672 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in fpsimd_to_sve()
677 	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));  in fpsimd_to_sve()
682  * Transfer the SVE state in task->thread.sve_state to
683  * task->thread.uw.fpsimd_state.
685  * Task can be a non-runnable task, or current.  In the latter case,
688  * task->thread.sve_state must point to at least sve_state_size(task)
690  * task->thread.sve_state must be up to date before calling this function.
692 static inline void sve_to_fpsimd(struct task_struct *task)  in sve_to_fpsimd()  argument
695 	void const *sst = task->thread.sve_state;  in sve_to_fpsimd()
696 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;  in sve_to_fpsimd()
703 	vl = thread_get_cur_vl(&task->thread);  in sve_to_fpsimd()
719 void task_smstop_sm(struct task_struct *task)  in task_smstop_sm()  argument
721 	if (!thread_sm_enabled(&task->thread))  in task_smstop_sm()
724 	__fpsimd_zero_vregs(&task->thread.uw.fpsimd_state);  in task_smstop_sm()
725 	task->thread.uw.fpsimd_state.fpsr = 0x0800009f;  in task_smstop_sm()
727 		task->thread.uw.fpmr = 0;  in task_smstop_sm()
729 	task->thread.svcr &= ~SVCR_SM_MASK;  in task_smstop_sm()
730 	task->thread.fp_type = FP_STATE_FPSIMD;  in task_smstop_sm()
740 static void sve_free(struct task_struct *task)  in sve_free()  argument
742 	kfree(task->thread.sve_state);  in sve_free()
743 	task->thread.sve_state = NULL;  in sve_free()
747  * Ensure that task->thread.sve_state is allocated and sufficiently large.
750  * task->thread.sve_state with new data.  The memory is always zeroed
754  * written previously be task.
756 void sve_alloc(struct task_struct *task, bool flush)  in sve_alloc()  argument
758 	if (task->thread.sve_state) {  in sve_alloc()
760 			memset(task->thread.sve_state, 0,  in sve_alloc()
761 			       sve_state_size(task));  in sve_alloc()
766 	task->thread.sve_state =  in sve_alloc()
767 		kzalloc(sve_state_size(task), GFP_KERNEL);  in sve_alloc()
771  * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the
772  * task's currently effective FPSIMD/SVE state.
774  * The task's FPSIMD/SVE/SME state must not be subject to concurrent
777 void fpsimd_sync_from_effective_state(struct task_struct *task)  in fpsimd_sync_from_effective_state()  argument
779 	if (task->thread.fp_type == FP_STATE_SVE)  in fpsimd_sync_from_effective_state()
780 		sve_to_fpsimd(task);  in fpsimd_sync_from_effective_state()
784  * Ensure that the task's currently effective FPSIMD/SVE state is up to date
785  * with respect to task->thread.uw.fpsimd_state, zeroing any effective
788  * The task's FPSIMD/SVE/SME state must not be subject to concurrent
791 void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)  in fpsimd_sync_to_effective_state_zeropad()  argument
794 	void *sst = task->thread.sve_state;  in fpsimd_sync_to_effective_state_zeropad()
795 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in fpsimd_sync_to_effective_state_zeropad()
797 	if (task->thread.fp_type != FP_STATE_SVE)  in fpsimd_sync_to_effective_state_zeropad()
800 	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));  in fpsimd_sync_to_effective_state_zeropad()
806 static int change_live_vector_length(struct task_struct *task,  in change_live_vector_length()  argument
810 	unsigned int sve_vl = task_get_sve_vl(task);  in change_live_vector_length()
811 	unsigned int sme_vl = task_get_sme_vl(task);  in change_live_vector_length()
822 	 * mutate the task's state in any way.  in change_live_vector_length()
839 	if (task == current)  in change_live_vector_length()
842 		fpsimd_flush_task_state(task);  in change_live_vector_length()
848 	fpsimd_sync_from_effective_state(task);  in change_live_vector_length()
849 	task_set_vl(task, type, vl);  in change_live_vector_length()
850 	kfree(task->thread.sve_state);  in change_live_vector_length()
851 	task->thread.sve_state = sve_state;  in change_live_vector_length()
852 	fpsimd_sync_to_effective_state_zeropad(task);  in change_live_vector_length()
855 		task->thread.svcr &= ~SVCR_ZA_MASK;  in change_live_vector_length()
856 		kfree(task->thread.sme_state);  in change_live_vector_length()
857 		task->thread.sme_state = sme_state;  in change_live_vector_length()
868 int vec_set_vector_length(struct task_struct *task, enum vec_type type,  in vec_set_vector_length()  argument
892 	if (!onexec && vl != task_get_vl(task, type)) {  in vec_set_vector_length()
893 		if (change_live_vector_length(task, type, vl))  in vec_set_vector_length()
898 		task_set_vl_onexec(task, type, vl);  in vec_set_vector_length()
901 		task_set_vl_onexec(task, type, 0);  in vec_set_vector_length()
903 	update_tsk_thread_flag(task, vec_vl_inherit_flag(type),  in vec_set_vector_length()
1207  * Ensure that task->thread.sme_state is allocated and sufficiently large.
1210  * task->thread.sme_state with new data.  The memory is always zeroed
1215 void sme_alloc(struct task_struct *task, bool flush)  in sme_alloc()  argument
1217 	if (task->thread.sme_state) {  in sme_alloc()
1219 			memset(task->thread.sme_state, 0,  in sme_alloc()
1220 			       sme_state_size(task));  in sme_alloc()
1225 	task->thread.sme_state =  in sme_alloc()
1226 		kzalloc(sme_state_size(task), GFP_KERNEL);  in sme_alloc()
1229 static void sme_free(struct task_struct *task)  in sme_free()  argument
1231 	kfree(task->thread.sme_state);  in sme_free()
1232 	task->thread.sme_state = NULL;  in sme_free()
1339 	 * update our metadata for the current task including  in sve_init_regs()
1389 	 * Even if the task can have used streaming mode we can only  in do_sve_acc()
1497 static void fpsimd_load_kernel_state(struct task_struct *task)  in fpsimd_load_kernel_state()  argument
1503 	 * FPSIMD context of the current task.  in fpsimd_load_kernel_state()
1505 	if (last->st == task->thread.kernel_fpsimd_state &&  in fpsimd_load_kernel_state()
1506 	    task->thread.kernel_fpsimd_cpu == smp_processor_id())  in fpsimd_load_kernel_state()
1509 	fpsimd_load_state(task->thread.kernel_fpsimd_state);  in fpsimd_load_kernel_state()
1512 static void fpsimd_save_kernel_state(struct task_struct *task)  in fpsimd_save_kernel_state()  argument
1515 		.st		= task->thread.kernel_fpsimd_state,  in fpsimd_save_kernel_state()
1521 	fpsimd_save_state(task->thread.kernel_fpsimd_state);  in fpsimd_save_kernel_state()
1524 	task->thread.kernel_fpsimd_cpu = smp_processor_id();  in fpsimd_save_kernel_state()
1528  * Invalidate any task's FPSIMD state that is present on this cpu.
1587 	 * Reset the task vector length as required.  This is where we  in fpsimd_flush_thread_vl()
1589 	 * configured: no kernel task can become a user task without  in fpsimd_flush_thread_vl()
1610 	 * If the task is not set to inherit, ensure that the vector  in fpsimd_flush_thread_vl()
1735 	 * TIF_FOREIGN_FPSTATE is set on the init task and copied by  in fpsimd_restore_current_state()
1779  * Invalidate live CPU copies of task t's FPSIMD state
1795 	 * reset the fpsimd_cpu for this task and clear the  in fpsimd_flush_task_state()
1845  * Task context in the FPSIMD registers is saved back to memory as necessary.
1853  * Unless called from non-preemptible task context, @state must point to a
1854  * caller provided buffer that will be used to preserve the task's kernel mode
1878 		 * will be context switched along with the rest of the task  in kernel_neon_begin()
1881 		 * On non-PREEMPT_RT, softirqs may interrupt task level kernel  in kernel_neon_begin()
1882 		 * mode FPSIMD, but the task will not be preemptible so setting  in kernel_neon_begin()
1884 		 * would mark the task context FPSIMD state as requiring a  in kernel_neon_begin()
1890 		 * mode in task context. So in this case, setting the flag here  in kernel_neon_begin()
1896 			 * FP/SIMD buffer for this task, so that the state can  in kernel_neon_begin()
1905 	/* Invalidate any task state remaining in the fpsimd regs: */  in kernel_neon_begin()
1913  * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1934 	 * the task context kernel mode FPSIMD state. This can only happen when  in kernel_neon_end()