18 static void pend_sync_exception(struct kvm_vcpu *vcpu)  in pend_sync_exception()  argument
21 	if (likely(!vcpu_has_nv(vcpu))) {  in pend_sync_exception()
22 		kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);  in pend_sync_exception()
32 	switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) {  in pend_sync_exception()
35 		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);  in pend_sync_exception()
39 		kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);  in pend_sync_exception()
42 		if (vcpu_el2_tge_is_set(vcpu))  in pend_sync_exception()
43 			kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);  in pend_sync_exception()
45 			kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);  in pend_sync_exception()
52 static bool match_target_el(struct kvm_vcpu *vcpu, unsigned long target)  in match_target_el()  argument
54 	return (vcpu_get_flag(vcpu, EXCEPT_MASK) == target);  in match_target_el()
57 static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)  in inject_abt64()  argument
59 	unsigned long cpsr = *vcpu_cpsr(vcpu);  in inject_abt64()
60 	bool is_aarch32 = vcpu_mode_is_32bit(vcpu);  in inject_abt64()
63 	pend_sync_exception(vcpu);  in inject_abt64()
69 	if (kvm_vcpu_trap_il_is32bit(vcpu))  in inject_abt64()
86 	if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC))) {  in inject_abt64()
87 		vcpu_write_sys_reg(vcpu, addr, FAR_EL1);  in inject_abt64()
88 		vcpu_write_sys_reg(vcpu, esr, ESR_EL1);  in inject_abt64()
90 		vcpu_write_sys_reg(vcpu, addr, FAR_EL2);  in inject_abt64()
91 		vcpu_write_sys_reg(vcpu, esr, ESR_EL2);  in inject_abt64()
95 static void inject_undef64(struct kvm_vcpu *vcpu)  in inject_undef64()  argument
99 	pend_sync_exception(vcpu);  in inject_undef64()
105 	if (kvm_vcpu_trap_il_is32bit(vcpu))  in inject_undef64()
108 	if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC)))  in inject_undef64()
109 		vcpu_write_sys_reg(vcpu, esr, ESR_EL1);  in inject_undef64()
111 		vcpu_write_sys_reg(vcpu, esr, ESR_EL2);  in inject_undef64()
119 static void inject_undef32(struct kvm_vcpu *vcpu)  in inject_undef32()  argument
121 	kvm_pend_exception(vcpu, EXCEPT_AA32_UND);  in inject_undef32()
128 static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)  in inject_abt32()  argument
134 	if (vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE) {  in inject_abt32()
141 	far = vcpu_read_sys_reg(vcpu, FAR_EL1);  in inject_abt32()
144 		kvm_pend_exception(vcpu, EXCEPT_AA32_IABT);  in inject_abt32()
147 		vcpu_write_sys_reg(vcpu, fsr, IFSR32_EL2);  in inject_abt32()
149 		kvm_pend_exception(vcpu, EXCEPT_AA32_DABT);  in inject_abt32()
152 		vcpu_write_sys_reg(vcpu, fsr, ESR_EL1);  in inject_abt32()
155 	vcpu_write_sys_reg(vcpu, far, FAR_EL1);  in inject_abt32()
160  * @vcpu: The VCPU to receive the data abort
163  * It is assumed that this code is called from the VCPU thread and that the
164  * VCPU therefore is not currently executing guest code.
166 void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)  in kvm_inject_dabt()  argument
168 	if (vcpu_el1_is_32bit(vcpu))  in kvm_inject_dabt()
169 		inject_abt32(vcpu, false, addr);  in kvm_inject_dabt()
171 		inject_abt64(vcpu, false, addr);  in kvm_inject_dabt()
176  * @vcpu: The VCPU to receive the prefetch abort
179  * It is assumed that this code is called from the VCPU thread and that the
180  * VCPU therefore is not currently executing guest code.
182 void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)  in kvm_inject_pabt()  argument
184 	if (vcpu_el1_is_32bit(vcpu))  in kvm_inject_pabt()
185 		inject_abt32(vcpu, true, addr);  in kvm_inject_pabt()
187 		inject_abt64(vcpu, true, addr);  in kvm_inject_pabt()
190 void kvm_inject_size_fault(struct kvm_vcpu *vcpu)  in kvm_inject_size_fault()  argument
194 	addr  = kvm_vcpu_get_fault_ipa(vcpu);  in kvm_inject_size_fault()
195 	addr |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);  in kvm_inject_size_fault()
197 	if (kvm_vcpu_trap_is_iabt(vcpu))  in kvm_inject_size_fault()
198 		kvm_inject_pabt(vcpu, addr);  in kvm_inject_size_fault()
200 		kvm_inject_dabt(vcpu, addr);  in kvm_inject_size_fault()
209 	if (vcpu_el1_is_32bit(vcpu) &&  in kvm_inject_size_fault()
210 	    !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))  in kvm_inject_size_fault()
213 	esr = vcpu_read_sys_reg(vcpu, ESR_EL1);  in kvm_inject_size_fault()
215 	vcpu_write_sys_reg(vcpu, esr, ESR_EL1);  in kvm_inject_size_fault()
220  * @vcpu: The vCPU in which to inject the exception
222  * It is assumed that this code is called from the VCPU thread and that the
223  * VCPU therefore is not currently executing guest code.
225 void kvm_inject_undefined(struct kvm_vcpu *vcpu)  in kvm_inject_undefined()  argument
227 	if (vcpu_el1_is_32bit(vcpu))  in kvm_inject_undefined()
228 		inject_undef32(vcpu);  in kvm_inject_undefined()
230 		inject_undef64(vcpu);  in kvm_inject_undefined()
233 void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 esr)  in kvm_set_sei_esr()  argument
235 	vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);  in kvm_set_sei_esr()
236 	*vcpu_hcr(vcpu) |= HCR_VSE;  in kvm_set_sei_esr()
241  * @vcpu: The VCPU to receive the exception
243  * It is assumed that this code is called from the VCPU thread and that the
244  * VCPU therefore is not currently executing guest code.
251 void kvm_inject_vabt(struct kvm_vcpu *vcpu)  in kvm_inject_vabt()  argument
253 	kvm_set_sei_esr(vcpu, ESR_ELx_ISV);  in kvm_inject_vabt()