369 		switch (current->thread.fp_type) {  in task_fpsimd_load()
375 			if (!thread_sm_enabled(&current->thread))  in task_fpsimd_load()
379 				sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);  in task_fpsimd_load()
401 		unsigned long sme_vl = task_get_sme_vl(current);  in task_fpsimd_load()
407 		write_sysreg_s(current->thread.svcr, SYS_SVCR);  in task_fpsimd_load()
409 		if (thread_za_enabled(&current->thread))  in task_fpsimd_load()
410 			sme_load_state(current->thread.sme_state,  in task_fpsimd_load()
413 		if (thread_sm_enabled(&current->thread))  in task_fpsimd_load()
418 		write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);  in task_fpsimd_load()
421 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);  in task_fpsimd_load()
422 		sve_load_state(sve_pffr(&current->thread),  in task_fpsimd_load()
423 			       &current->thread.uw.fpsimd_state.fpsr,  in task_fpsimd_load()
426 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);  in task_fpsimd_load()
427 		fpsimd_load_state(&current->thread.uw.fpsimd_state);  in task_fpsimd_load()
832 	if (task == current)  in change_live_vector_length()
914 		ret = task_get_vl_onexec(current, type);  in vec_prctl_status()
916 		ret = task_get_vl(current, type);  in vec_prctl_status()
936 	ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags);  in sve_set_current_vl()
965 	ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags);  in sme_set_current_vl()
1308 			sve_vq_from_vl(task_get_sve_vl(current)) - 1;  in sve_init_regs()
1313 		fpsimd_to_sve(current);  in sve_init_regs()
1314 		current->thread.fp_type = FP_STATE_SVE;  in sve_init_regs()
1315 		fpsimd_flush_task_state(current);  in sve_init_regs()
1338 	sve_alloc(current, true);  in do_sve_acc()
1339 	if (!current->thread.sve_state) {  in do_sve_acc()
1389 	sve_alloc(current, false);  in do_sme_acc()
1390 	sme_alloc(current, true);  in do_sme_acc()
1391 	if (!current->thread.sve_state || !current->thread.sme_state) {  in do_sme_acc()
1404 			sve_vq_from_vl(task_get_sme_vl(current)) - 1;  in do_sme_acc()
1409 		fpsimd_flush_task_state(current);  in do_sme_acc()
1455 		       current);  in do_fpsimd_exc()
1520 		fpsimd_save_kernel_state(current);  in fpsimd_thread_switch()
1557 	vl = task_get_vl_onexec(current, type);  in fpsimd_flush_thread_vl()
1568 	task_set_vl(current, type, vl);  in fpsimd_flush_thread_vl()
1575 		task_set_vl_onexec(current, type, 0);  in fpsimd_flush_thread_vl()
1588 	fpsimd_flush_task_state(current);  in fpsimd_flush_thread()
1589 	memset(&current->thread.uw.fpsimd_state, 0,  in fpsimd_flush_thread()
1590 	       sizeof(current->thread.uw.fpsimd_state));  in fpsimd_flush_thread()
1596 		sve_state = current->thread.sve_state;  in fpsimd_flush_thread()
1597 		current->thread.sve_state = NULL;  in fpsimd_flush_thread()
1606 		sme_state = current->thread.sme_state;  in fpsimd_flush_thread()
1607 		current->thread.sme_state = NULL;  in fpsimd_flush_thread()
1610 		current->thread.svcr = 0;  in fpsimd_flush_thread()
1614 		current->thread.uw.fpmr = 0;  in fpsimd_flush_thread()
1616 	current->thread.fp_type = FP_STATE_FPSIMD;  in fpsimd_flush_thread()
1647 	last->st = &current->thread.uw.fpsimd_state;  in fpsimd_bind_task_to_cpu()
1648 	last->sve_state = current->thread.sve_state;  in fpsimd_bind_task_to_cpu()
1649 	last->sme_state = current->thread.sme_state;  in fpsimd_bind_task_to_cpu()
1650 	last->sve_vl = task_get_sve_vl(current);  in fpsimd_bind_task_to_cpu()
1651 	last->sme_vl = task_get_sme_vl(current);  in fpsimd_bind_task_to_cpu()
1652 	last->svcr = &current->thread.svcr;  in fpsimd_bind_task_to_cpu()
1653 	last->fpmr = &current->thread.uw.fpmr;  in fpsimd_bind_task_to_cpu()
1654 	last->fp_type = &current->thread.fp_type;  in fpsimd_bind_task_to_cpu()
1656 	current->thread.fpsimd_cpu = smp_processor_id();  in fpsimd_bind_task_to_cpu()
1734 	current->thread.uw.fpsimd_state = *state;  in fpsimd_update_current_state()
1735 	if (current->thread.fp_type == FP_STATE_SVE)  in fpsimd_update_current_state()
1736 		fpsimd_to_sve(current);  in fpsimd_update_current_state()
1774 	fpsimd_flush_task_state(current);  in fpsimd_save_and_flush_current_state()
1860 			WARN_ON(current->thread.kernel_fpsimd_state != NULL);  in kernel_neon_begin()
1861 			current->thread.kernel_fpsimd_state = state;  in kernel_neon_begin()
1902 		WARN_ON(current->thread.kernel_fpsimd_state != state);  in kernel_neon_end()
1903 		current->thread.kernel_fpsimd_state = NULL;  in kernel_neon_end()