79 static void kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)  in kvm_vcpu_enable_sve()  argument
81 	vcpu->arch.sve_max_vl = kvm_sve_max_vl;  in kvm_vcpu_enable_sve()
88 	set_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &vcpu->kvm->arch.flags);  in kvm_vcpu_enable_sve()
95 static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu)  in kvm_vcpu_finalize_sve()  argument
102 	vl = vcpu->arch.sve_max_vl;  in kvm_vcpu_finalize_sve()
113 	reg_sz = vcpu_sve_state_size(vcpu);  in kvm_vcpu_finalize_sve()
124 	vcpu->arch.sve_state = buf;  in kvm_vcpu_finalize_sve()
125 	vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);  in kvm_vcpu_finalize_sve()
129 int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)  in kvm_arm_vcpu_finalize()  argument
133 		if (!vcpu_has_sve(vcpu))  in kvm_arm_vcpu_finalize()
136 		if (kvm_arm_vcpu_sve_finalized(vcpu))  in kvm_arm_vcpu_finalize()
139 		return kvm_vcpu_finalize_sve(vcpu);  in kvm_arm_vcpu_finalize()
145 bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)  in kvm_arm_vcpu_is_finalized()  argument
147 	if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))  in kvm_arm_vcpu_is_finalized()
153 void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arm_vcpu_destroy()  argument
155 	void *sve_state = vcpu->arch.sve_state;  in kvm_arm_vcpu_destroy()
157 	kvm_unshare_hyp(vcpu, vcpu + 1);  in kvm_arm_vcpu_destroy()
159 		kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));  in kvm_arm_vcpu_destroy()
161 	free_page((unsigned long)vcpu->arch.ctxt.vncr_array);  in kvm_arm_vcpu_destroy()
162 	kfree(vcpu->arch.vncr_tlb);  in kvm_arm_vcpu_destroy()
163 	kfree(vcpu->arch.ccsidr);  in kvm_arm_vcpu_destroy()
166 static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)  in kvm_vcpu_reset_sve()  argument
168 	if (vcpu_has_sve(vcpu))  in kvm_vcpu_reset_sve()
169 		memset(vcpu->arch.sve_state, 0, vcpu_sve_state_size(vcpu));  in kvm_vcpu_reset_sve()
190 void kvm_reset_vcpu(struct kvm_vcpu *vcpu)  in kvm_reset_vcpu()  argument
196 	spin_lock(&vcpu->arch.mp_state_lock);  in kvm_reset_vcpu()
197 	reset_state = vcpu->arch.reset_state;  in kvm_reset_vcpu()
198 	vcpu->arch.reset_state.reset = false;  in kvm_reset_vcpu()
199 	spin_unlock(&vcpu->arch.mp_state_lock);  in kvm_reset_vcpu()
202 	loaded = (vcpu->cpu != -1);  in kvm_reset_vcpu()
204 		kvm_arch_vcpu_put(vcpu);  in kvm_reset_vcpu()
206 	if (!kvm_arm_vcpu_sve_finalized(vcpu)) {  in kvm_reset_vcpu()
207 		if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE))  in kvm_reset_vcpu()
208 			kvm_vcpu_enable_sve(vcpu);  in kvm_reset_vcpu()
210 		kvm_vcpu_reset_sve(vcpu);  in kvm_reset_vcpu()
213 	if (vcpu_el1_is_32bit(vcpu))  in kvm_reset_vcpu()
215 	else if (vcpu_has_nv(vcpu))  in kvm_reset_vcpu()
221 	memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu)));  in kvm_reset_vcpu()
222 	memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs));  in kvm_reset_vcpu()
223 	vcpu->arch.ctxt.spsr_abt = 0;  in kvm_reset_vcpu()
224 	vcpu->arch.ctxt.spsr_und = 0;  in kvm_reset_vcpu()
225 	vcpu->arch.ctxt.spsr_irq = 0;  in kvm_reset_vcpu()
226 	vcpu->arch.ctxt.spsr_fiq = 0;  in kvm_reset_vcpu()
227 	vcpu_gp_regs(vcpu)->pstate = pstate;  in kvm_reset_vcpu()
230 	kvm_reset_sys_regs(vcpu);  in kvm_reset_vcpu()
240 		if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {  in kvm_reset_vcpu()
242 			vcpu_set_thumb(vcpu);  in kvm_reset_vcpu()
247 			kvm_vcpu_set_be(vcpu);  in kvm_reset_vcpu()
249 		*vcpu_pc(vcpu) = target_pc;  in kvm_reset_vcpu()
261 		vcpu_clear_flag(vcpu, PENDING_EXCEPTION);  in kvm_reset_vcpu()
262 		vcpu_clear_flag(vcpu, EXCEPT_MASK);  in kvm_reset_vcpu()
263 		vcpu_clear_flag(vcpu, INCREMENT_PC);  in kvm_reset_vcpu()
264 		vcpu_set_reg(vcpu, 0, reset_state.r0);  in kvm_reset_vcpu()
268 	kvm_timer_vcpu_reset(vcpu);  in kvm_reset_vcpu()
271 		kvm_arch_vcpu_load(vcpu, smp_processor_id());  in kvm_reset_vcpu()