257 	return task->thread.vl[type];  in task_get_vl()
263 	task->thread.vl[type] = vl;  in task_set_vl()
269 	return task->thread.vl_onexec[type];  in task_get_vl_onexec()
275 	task->thread.vl_onexec[type] = vl;  in task_set_vl_onexec()
369 		switch (current->thread.fp_type) {  in task_fpsimd_load()
375 			if (!thread_sm_enabled(&current->thread))  in task_fpsimd_load()
407 		write_sysreg_s(current->thread.svcr, SYS_SVCR);  in task_fpsimd_load()
409 		if (thread_za_enabled(&current->thread))  in task_fpsimd_load()
410 			sme_load_state(current->thread.sme_state,  in task_fpsimd_load()
413 		if (thread_sm_enabled(&current->thread))  in task_fpsimd_load()
418 		write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);  in task_fpsimd_load()
421 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);  in task_fpsimd_load()
422 		sve_load_state(sve_pffr(&current->thread),  in task_fpsimd_load()
423 			       &current->thread.uw.fpsimd_state.fpsr,  in task_fpsimd_load()
426 		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);  in task_fpsimd_load()
427 		fpsimd_load_state(&current->thread.uw.fpsimd_state);  in task_fpsimd_load()
664 	void *sst = task->thread.sve_state;  in fpsimd_to_sve()
665 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in fpsimd_to_sve()
670 	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));  in fpsimd_to_sve()
688 	void const *sst = task->thread.sve_state;  in sve_to_fpsimd()
689 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;  in sve_to_fpsimd()
696 	vl = thread_get_cur_vl(&task->thread);  in sve_to_fpsimd()
714 	if (!thread_sm_enabled(&task->thread))  in task_smstop_sm()
717 	__fpsimd_zero_vregs(&task->thread.uw.fpsimd_state);  in task_smstop_sm()
718 	task->thread.uw.fpsimd_state.fpsr = 0x0800009f;  in task_smstop_sm()
720 		task->thread.uw.fpmr = 0;  in task_smstop_sm()
722 	task->thread.svcr &= ~SVCR_SM_MASK;  in task_smstop_sm()
723 	task->thread.fp_type = FP_STATE_FPSIMD;  in task_smstop_sm()
735 	kfree(task->thread.sve_state);  in sve_free()
736 	task->thread.sve_state = NULL;  in sve_free()
751 	if (task->thread.sve_state) {  in sve_alloc()
753 			memset(task->thread.sve_state, 0,  in sve_alloc()
759 	task->thread.sve_state =  in sve_alloc()
772 	if (task->thread.fp_type == FP_STATE_SVE)  in fpsimd_sync_from_effective_state()
787 	void *sst = task->thread.sve_state;  in fpsimd_sync_to_effective_state_zeropad()
788 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in fpsimd_sync_to_effective_state_zeropad()
790 	if (task->thread.fp_type != FP_STATE_SVE)  in fpsimd_sync_to_effective_state_zeropad()
793 	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));  in fpsimd_sync_to_effective_state_zeropad()
843 	kfree(task->thread.sve_state);  in change_live_vector_length()
844 	task->thread.sve_state = sve_state;  in change_live_vector_length()
848 		task->thread.svcr &= ~SVCR_ZA_MASK;  in change_live_vector_length()
849 		kfree(task->thread.sme_state);  in change_live_vector_length()
850 		task->thread.sme_state = sme_state;  in change_live_vector_length()
1178 	if (task->thread.sme_state) {  in sme_alloc()
1180 			memset(task->thread.sme_state, 0,  in sme_alloc()
1186 	task->thread.sme_state =  in sme_alloc()
1192 	kfree(task->thread.sme_state);  in sme_free()
1193 	task->thread.sme_state = NULL;  in sme_free()
1314 		current->thread.fp_type = FP_STATE_SVE;  in sve_init_regs()
1339 	if (!current->thread.sve_state) {  in do_sve_acc()
1391 	if (!current->thread.sve_state || !current->thread.sme_state) {  in do_sme_acc()
1466 	if (last->st == task->thread.kernel_fpsimd_state &&  in fpsimd_load_kernel_state()
1467 	    task->thread.kernel_fpsimd_cpu == smp_processor_id())  in fpsimd_load_kernel_state()
1470 	fpsimd_load_state(task->thread.kernel_fpsimd_state);  in fpsimd_load_kernel_state()
1476 		.st		= task->thread.kernel_fpsimd_state,  in fpsimd_save_kernel_state()
1482 	fpsimd_save_state(task->thread.kernel_fpsimd_state);  in fpsimd_save_kernel_state()
1485 	task->thread.kernel_fpsimd_cpu = smp_processor_id();  in fpsimd_save_kernel_state()
1535 			&next->thread.uw.fpsimd_state;  in fpsimd_thread_switch()
1536 		wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();  in fpsimd_thread_switch()
1589 	memset(&current->thread.uw.fpsimd_state, 0,  in fpsimd_flush_thread()
1590 	       sizeof(current->thread.uw.fpsimd_state));  in fpsimd_flush_thread()
1596 		sve_state = current->thread.sve_state;  in fpsimd_flush_thread()
1597 		current->thread.sve_state = NULL;  in fpsimd_flush_thread()
1606 		sme_state = current->thread.sme_state;  in fpsimd_flush_thread()
1607 		current->thread.sme_state = NULL;  in fpsimd_flush_thread()
1610 		current->thread.svcr = 0;  in fpsimd_flush_thread()
1614 		current->thread.uw.fpmr = 0;  in fpsimd_flush_thread()
1616 	current->thread.fp_type = FP_STATE_FPSIMD;  in fpsimd_flush_thread()
1647 	last->st = &current->thread.uw.fpsimd_state;  in fpsimd_bind_task_to_cpu()
1648 	last->sve_state = current->thread.sve_state;  in fpsimd_bind_task_to_cpu()
1649 	last->sme_state = current->thread.sme_state;  in fpsimd_bind_task_to_cpu()
1652 	last->svcr = &current->thread.svcr;  in fpsimd_bind_task_to_cpu()
1653 	last->fpmr = &current->thread.uw.fpmr;  in fpsimd_bind_task_to_cpu()
1654 	last->fp_type = &current->thread.fp_type;  in fpsimd_bind_task_to_cpu()
1656 	current->thread.fpsimd_cpu = smp_processor_id();  in fpsimd_bind_task_to_cpu()
1734 	current->thread.uw.fpsimd_state = *state;  in fpsimd_update_current_state()
1735 	if (current->thread.fp_type == FP_STATE_SVE)  in fpsimd_update_current_state()
1752 	t->thread.fpsimd_cpu = NR_CPUS;  in fpsimd_flush_task_state()
1753 	t->thread.kernel_fpsimd_state = NULL;  in fpsimd_flush_task_state()
1860 			WARN_ON(current->thread.kernel_fpsimd_state != NULL);  in kernel_neon_begin()
1861 			current->thread.kernel_fpsimd_state = state;  in kernel_neon_begin()
1902 		WARN_ON(current->thread.kernel_fpsimd_state != state);  in kernel_neon_end()
1903 		current->thread.kernel_fpsimd_state = NULL;  in kernel_neon_end()