67  * the id of the current CPU every time the state is loaded onto a CPU. For (b),
79  * indicate whether or not the userland FPSIMD state of the current task is
81  * CPU currently contain the most recent userland FPSIMD state of the current
98  *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
104  *   fpsimd_cpu field is set to the id of the current CPU, the current
353  * Update current's FPSIMD/SVE registers from thread_struct.
369 		switch (current->thread.fp_type) {  in task_fpsimd_load()
375 			if (!thread_sm_enabled(&current->thread))  in task_fpsimd_load()
379 				sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);  in task_fpsimd_load()
401 		unsigned long sme_vl = task_get_sme_vl(current);  in task_fpsimd_load()
407 		write_sysreg_s(current->thread.svcr, SYS_SVCR);  in task_fpsimd_load()
409 		if (thread_za_enabled(&current->thread))  in task_fpsimd_load()
410 			sme_load_state(current->thread.sme_state,  in task_fpsimd_load()
413 		if (thread_sm_enabled(&current->thread))  in task_fpsimd_load()
418 		write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);  in task_fpsimd_load()
421 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);  in task_fpsimd_load()
422 		sve_load_state(sve_pffr(&current->thread),  in task_fpsimd_load()
423 			       &current->thread.uw.fpsimd_state.fpsr,  in task_fpsimd_load()
426 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);  in task_fpsimd_load()
427 		fpsimd_load_state(&current->thread.uw.fpsimd_state);  in task_fpsimd_load()
434  * current context is the context last bound to the CPU stored in
436  * than the host thread for the VM pointed to by current. This means
438  * than via current, if we are saving KVM state then it will have
496 			 * Can't save the user regs, so current would  in fpsimd_save_user_state()
653  * Task can be a non-runnable task, or current.  In the latter case,
678  * Task can be a non-runnable task, or current.  In the latter case,
832 	if (task == current)  in change_live_vector_length()
903  * Encode the current vector length and flags for return.
914 		ret = task_get_vl_onexec(current, type);  in vec_prctl_status()
916 		ret = task_get_vl(current, type);  in vec_prctl_status()
936 	ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags);  in sve_set_current_vl()
965 	ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags);  in sme_set_current_vl()
1026  * those not supported by the current CPU.
1041  * Check whether the current CPU supports all VQs in the committed set.
1298 	 * is not shared with FPSIMD. If (as is likely) the current  in sve_init_regs()
1300 	 * update our metadata for the current task including  in sve_init_regs()
1308 			sve_vq_from_vl(task_get_sve_vl(current)) - 1;  in sve_init_regs()
1313 		fpsimd_to_sve(current);  in sve_init_regs()
1314 		current->thread.fp_type = FP_STATE_SVE;  in sve_init_regs()
1315 		fpsimd_flush_task_state(current);  in sve_init_regs()
1322  * Storage is allocated for the full SVE state, the current FPSIMD
1338 	sve_alloc(current, true);  in do_sve_acc()
1339 	if (!current->thread.sve_state) {  in do_sve_acc()
1364  * Storage is allocated for the full SVE and SME state, the current
1389 	sve_alloc(current, false);  in do_sme_acc()
1390 	sme_alloc(current, true);  in do_sme_acc()
1391 	if (!current->thread.sve_state || !current->thread.sme_state) {  in do_sme_acc()
1404 			sve_vq_from_vl(task_get_sme_vl(current)) - 1;  in do_sme_acc()
1409 		fpsimd_flush_task_state(current);  in do_sme_acc()
1434  * Raise a SIGFPE for the current process.
1455 		       current);  in do_fpsimd_exc()
1464 	 * FPSIMD context of the current task.  in fpsimd_load_kernel_state()
1520 		fpsimd_save_kernel_state(current);  in fpsimd_thread_switch()
1557 	vl = task_get_vl_onexec(current, type);  in fpsimd_flush_thread_vl()
1568 	task_set_vl(current, type, vl);  in fpsimd_flush_thread_vl()
1575 		task_set_vl_onexec(current, type, 0);  in fpsimd_flush_thread_vl()
1588 	fpsimd_flush_task_state(current);  in fpsimd_flush_thread()
1589 	memset(&current->thread.uw.fpsimd_state, 0,  in fpsimd_flush_thread()
1590 	       sizeof(current->thread.uw.fpsimd_state));  in fpsimd_flush_thread()
1596 		sve_state = current->thread.sve_state;  in fpsimd_flush_thread()
1597 		current->thread.sve_state = NULL;  in fpsimd_flush_thread()
1606 		sme_state = current->thread.sme_state;  in fpsimd_flush_thread()
1607 		current->thread.sme_state = NULL;  in fpsimd_flush_thread()
1610 		current->thread.svcr = 0;  in fpsimd_flush_thread()
1614 		current->thread.uw.fpmr = 0;  in fpsimd_flush_thread()
1616 	current->thread.fp_type = FP_STATE_FPSIMD;  in fpsimd_flush_thread()
1624  * Save the userland FPSIMD state of 'current' to memory, but only if the state
1625  * currently held in the registers does in fact belong to 'current'
1638  * Associate current's FPSIMD context with this cpu
1647 	last->st = &current->thread.uw.fpsimd_state;  in fpsimd_bind_task_to_cpu()
1648 	last->sve_state = current->thread.sve_state;  in fpsimd_bind_task_to_cpu()
1649 	last->sme_state = current->thread.sme_state;  in fpsimd_bind_task_to_cpu()
1650 	last->sve_vl = task_get_sve_vl(current);  in fpsimd_bind_task_to_cpu()
1651 	last->sme_vl = task_get_sme_vl(current);  in fpsimd_bind_task_to_cpu()
1652 	last->svcr = &current->thread.svcr;  in fpsimd_bind_task_to_cpu()
1653 	last->fpmr = &current->thread.uw.fpmr;  in fpsimd_bind_task_to_cpu()
1654 	last->fp_type = &current->thread.fp_type;  in fpsimd_bind_task_to_cpu()
1656 	current->thread.fpsimd_cpu = smp_processor_id();  in fpsimd_bind_task_to_cpu()
1688  * Load the userland FPSIMD state of 'current' from memory, but only if the
1690  * state of 'current'.  This is called when we are preparing to return to
1734 	current->thread.uw.fpsimd_state = *state;  in fpsimd_update_current_state()
1735 	if (current->thread.fp_type == FP_STATE_SVE)  in fpsimd_update_current_state()
1736 		fpsimd_to_sve(current);  in fpsimd_update_current_state()
1774 	fpsimd_flush_task_state(current);  in fpsimd_save_and_flush_current_state()
1860 			WARN_ON(current->thread.kernel_fpsimd_state != NULL);  in kernel_neon_begin()
1861 			current->thread.kernel_fpsimd_state = state;  in kernel_neon_begin()
1874  * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1902 		WARN_ON(current->thread.kernel_fpsimd_state != state);  in kernel_neon_end()
1903 		current->thread.kernel_fpsimd_state = NULL;  in kernel_neon_end()
1943 		 * expect the current task to ever return to user space again,  in __efi_fpsimd_begin()