xref: /linux/arch/arm64/include/asm/kvm_emulate.h (revision 5d6ba5ab8582aa35c1ee98e47af28e6f6772596c)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2012,2013 - ARM Ltd
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  *
6  * Derived from arch/arm/include/kvm_emulate.h
7  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
9  */
10 
11 #ifndef __ARM64_KVM_EMULATE_H__
12 #define __ARM64_KVM_EMULATE_H__
13 
14 #include <linux/bitfield.h>
15 #include <linux/kvm_host.h>
16 
17 #include <asm/debug-monitors.h>
18 #include <asm/esr.h>
19 #include <asm/kvm_arm.h>
20 #include <asm/kvm_hyp.h>
21 #include <asm/kvm_nested.h>
22 #include <asm/ptrace.h>
23 #include <asm/cputype.h>
24 #include <asm/virt.h>
25 
26 #define CURRENT_EL_SP_EL0_VECTOR	0x0
27 #define CURRENT_EL_SP_ELx_VECTOR	0x200
28 #define LOWER_EL_AArch64_VECTOR		0x400
29 #define LOWER_EL_AArch32_VECTOR		0x600
30 
31 enum exception_type {
32 	except_type_sync	= 0,
33 	except_type_irq		= 0x80,
34 	except_type_fiq		= 0x100,
35 	except_type_serror	= 0x180,
36 };
37 
38 #define kvm_exception_type_names		\
39 	{ except_type_sync,	"SYNC"   },	\
40 	{ except_type_irq,	"IRQ"    },	\
41 	{ except_type_fiq,	"FIQ"    },	\
42 	{ except_type_serror,	"SERROR" }
43 
44 bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
45 void kvm_skip_instr32(struct kvm_vcpu *vcpu);
46 
47 void kvm_inject_undefined(struct kvm_vcpu *vcpu);
48 void kvm_inject_vabt(struct kvm_vcpu *vcpu);
49 void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
50 void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
51 void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
52 
53 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
54 
55 void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
56 int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
57 int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
58 
kvm_inject_nested_sve_trap(struct kvm_vcpu * vcpu)59 static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
60 {
61 	u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) |
62 		  ESR_ELx_IL;
63 
64 	kvm_inject_nested_sync(vcpu, esr);
65 }
66 
67 #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
vcpu_el1_is_32bit(struct kvm_vcpu * vcpu)68 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
69 {
70 	return !(vcpu->arch.hcr_el2 & HCR_RW);
71 }
72 #else
vcpu_el1_is_32bit(struct kvm_vcpu * vcpu)73 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
74 {
75 	return vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT);
76 }
77 #endif
78 
vcpu_reset_hcr(struct kvm_vcpu * vcpu)79 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
80 {
81 	if (!vcpu_has_run_once(vcpu))
82 		vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
83 
84 	/*
85 	 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C
86 	 * get set in SCTLR_EL1 such that we can detect when the guest
87 	 * MMU gets turned on and do the necessary cache maintenance
88 	 * then.
89 	 */
90 	if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
91 		vcpu->arch.hcr_el2 |= HCR_TVM;
92 }
93 
vcpu_hcr(struct kvm_vcpu * vcpu)94 static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
95 {
96 	return (unsigned long *)&vcpu->arch.hcr_el2;
97 }
98 
vcpu_clear_wfx_traps(struct kvm_vcpu * vcpu)99 static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu)
100 {
101 	vcpu->arch.hcr_el2 &= ~HCR_TWE;
102 	if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
103 	    vcpu->kvm->arch.vgic.nassgireq)
104 		vcpu->arch.hcr_el2 &= ~HCR_TWI;
105 	else
106 		vcpu->arch.hcr_el2 |= HCR_TWI;
107 }
108 
vcpu_set_wfx_traps(struct kvm_vcpu * vcpu)109 static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
110 {
111 	vcpu->arch.hcr_el2 |= HCR_TWE;
112 	vcpu->arch.hcr_el2 |= HCR_TWI;
113 }
114 
vcpu_get_vsesr(struct kvm_vcpu * vcpu)115 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
116 {
117 	return vcpu->arch.vsesr_el2;
118 }
119 
vcpu_set_vsesr(struct kvm_vcpu * vcpu,u64 vsesr)120 static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
121 {
122 	vcpu->arch.vsesr_el2 = vsesr;
123 }
124 
vcpu_pc(const struct kvm_vcpu * vcpu)125 static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
126 {
127 	return (unsigned long *)&vcpu_gp_regs(vcpu)->pc;
128 }
129 
vcpu_cpsr(const struct kvm_vcpu * vcpu)130 static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
131 {
132 	return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate;
133 }
134 
vcpu_mode_is_32bit(const struct kvm_vcpu * vcpu)135 static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
136 {
137 	return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
138 }
139 
kvm_condition_valid(const struct kvm_vcpu * vcpu)140 static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
141 {
142 	if (vcpu_mode_is_32bit(vcpu))
143 		return kvm_condition_valid32(vcpu);
144 
145 	return true;
146 }
147 
vcpu_set_thumb(struct kvm_vcpu * vcpu)148 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
149 {
150 	*vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
151 }
152 
153 /*
154  * vcpu_get_reg and vcpu_set_reg should always be passed a register number
155  * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
156  * AArch32 with banked registers.
157  */
vcpu_get_reg(const struct kvm_vcpu * vcpu,u8 reg_num)158 static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
159 					 u8 reg_num)
160 {
161 	return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num];
162 }
163 
vcpu_set_reg(struct kvm_vcpu * vcpu,u8 reg_num,unsigned long val)164 static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
165 				unsigned long val)
166 {
167 	if (reg_num != 31)
168 		vcpu_gp_regs(vcpu)->regs[reg_num] = val;
169 }
170 
vcpu_is_el2_ctxt(const struct kvm_cpu_context * ctxt)171 static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt)
172 {
173 	switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) {
174 	case PSR_MODE_EL2h:
175 	case PSR_MODE_EL2t:
176 		return true;
177 	default:
178 		return false;
179 	}
180 }
181 
vcpu_is_el2(const struct kvm_vcpu * vcpu)182 static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu)
183 {
184 	return vcpu_is_el2_ctxt(&vcpu->arch.ctxt);
185 }
186 
vcpu_el2_e2h_is_set(const struct kvm_vcpu * vcpu)187 static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu)
188 {
189 	return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) ||
190 		(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H));
191 }
192 
vcpu_el2_tge_is_set(const struct kvm_vcpu * vcpu)193 static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu)
194 {
195 	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
196 }
197 
is_hyp_ctxt(const struct kvm_vcpu * vcpu)198 static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
199 {
200 	bool e2h, tge;
201 	u64 hcr;
202 
203 	if (!vcpu_has_nv(vcpu))
204 		return false;
205 
206 	hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
207 
208 	e2h = (hcr & HCR_E2H);
209 	tge = (hcr & HCR_TGE);
210 
211 	/*
212 	 * We are in a hypervisor context if the vcpu mode is EL2 or
213 	 * E2H and TGE bits are set. The latter means we are in the user space
214 	 * of the VHE kernel. ARMv8.1 ARM describes this as 'InHost'
215 	 *
216 	 * Note that the HCR_EL2.{E2H,TGE}={0,1} isn't really handled in the
217 	 * rest of the KVM code, and will result in a misbehaving guest.
218 	 */
219 	return vcpu_is_el2(vcpu) || (e2h && tge) || tge;
220 }
221 
vcpu_is_host_el0(const struct kvm_vcpu * vcpu)222 static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu)
223 {
224 	return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);
225 }
226 
227 /*
228  * The layout of SPSR for an AArch32 state is different when observed from an
229  * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
230  * view given an AArch64 view.
231  *
232  * In ARM DDI 0487E.a see:
233  *
234  * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426
235  * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256
236  * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280
237  *
238  * Which show the following differences:
239  *
240  * | Bit | AA64 | AA32 | Notes                       |
241  * +-----+------+------+-----------------------------|
242  * | 24  | DIT  | J    | J is RES0 in ARMv8          |
243  * | 21  | SS   | DIT  | SS doesn't exist in AArch32 |
244  *
245  * ... and all other bits are (currently) common.
246  */
host_spsr_to_spsr32(unsigned long spsr)247 static inline unsigned long host_spsr_to_spsr32(unsigned long spsr)
248 {
249 	const unsigned long overlap = BIT(24) | BIT(21);
250 	unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT);
251 
252 	spsr &= ~overlap;
253 
254 	spsr |= dit << 21;
255 
256 	return spsr;
257 }
258 
vcpu_mode_priv(const struct kvm_vcpu * vcpu)259 static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
260 {
261 	u32 mode;
262 
263 	if (vcpu_mode_is_32bit(vcpu)) {
264 		mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
265 		return mode > PSR_AA32_MODE_USR;
266 	}
267 
268 	mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
269 
270 	return mode != PSR_MODE_EL0t;
271 }
272 
kvm_vcpu_get_esr(const struct kvm_vcpu * vcpu)273 static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu)
274 {
275 	return vcpu->arch.fault.esr_el2;
276 }
277 
guest_hyp_wfx_traps_enabled(const struct kvm_vcpu * vcpu)278 static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu)
279 {
280 	u64 esr = kvm_vcpu_get_esr(vcpu);
281 	bool is_wfe = !!(esr & ESR_ELx_WFx_ISS_WFE);
282 	u64 hcr_el2 = __vcpu_sys_reg(vcpu, HCR_EL2);
283 
284 	if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu))
285 		return false;
286 
287 	return ((is_wfe && (hcr_el2 & HCR_TWE)) ||
288 		(!is_wfe && (hcr_el2 & HCR_TWI)));
289 }
290 
kvm_vcpu_get_condition(const struct kvm_vcpu * vcpu)291 static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
292 {
293 	u64 esr = kvm_vcpu_get_esr(vcpu);
294 
295 	if (esr & ESR_ELx_CV)
296 		return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
297 
298 	return -1;
299 }
300 
kvm_vcpu_get_hfar(const struct kvm_vcpu * vcpu)301 static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
302 {
303 	return vcpu->arch.fault.far_el2;
304 }
305 
kvm_vcpu_get_fault_ipa(const struct kvm_vcpu * vcpu)306 static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
307 {
308 	return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
309 }
310 
kvm_vcpu_get_disr(const struct kvm_vcpu * vcpu)311 static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
312 {
313 	return vcpu->arch.fault.disr_el1;
314 }
315 
kvm_vcpu_hvc_get_imm(const struct kvm_vcpu * vcpu)316 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
317 {
318 	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK;
319 }
320 
kvm_vcpu_dabt_isvalid(const struct kvm_vcpu * vcpu)321 static __always_inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
322 {
323 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_ISV);
324 }
325 
kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu * vcpu)326 static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu)
327 {
328 	return kvm_vcpu_get_esr(vcpu) & (ESR_ELx_CM | ESR_ELx_WNR | ESR_ELx_FSC);
329 }
330 
kvm_vcpu_dabt_issext(const struct kvm_vcpu * vcpu)331 static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
332 {
333 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SSE);
334 }
335 
kvm_vcpu_dabt_issf(const struct kvm_vcpu * vcpu)336 static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu)
337 {
338 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SF);
339 }
340 
kvm_vcpu_dabt_get_rd(const struct kvm_vcpu * vcpu)341 static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
342 {
343 	return (kvm_vcpu_get_esr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
344 }
345 
kvm_vcpu_abt_iss1tw(const struct kvm_vcpu * vcpu)346 static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
347 {
348 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
349 }
350 
351 /* Always check for S1PTW *before* using this. */
kvm_vcpu_dabt_iswrite(const struct kvm_vcpu * vcpu)352 static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
353 {
354 	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
355 }
356 
kvm_vcpu_dabt_is_cm(const struct kvm_vcpu * vcpu)357 static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
358 {
359 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
360 }
361 
kvm_vcpu_dabt_get_as(const struct kvm_vcpu * vcpu)362 static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
363 {
364 	return 1 << ((kvm_vcpu_get_esr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
365 }
366 
367 /* This one is not specific to Data Abort */
kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu * vcpu)368 static __always_inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
369 {
370 	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_IL);
371 }
372 
kvm_vcpu_trap_get_class(const struct kvm_vcpu * vcpu)373 static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
374 {
375 	return ESR_ELx_EC(kvm_vcpu_get_esr(vcpu));
376 }
377 
kvm_vcpu_trap_is_iabt(const struct kvm_vcpu * vcpu)378 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
379 {
380 	return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
381 }
382 
kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu * vcpu)383 static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
384 {
385 	return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
386 }
387 
kvm_vcpu_trap_get_fault(const struct kvm_vcpu * vcpu)388 static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
389 {
390 	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC;
391 }
392 
393 static inline
kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu * vcpu)394 bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu)
395 {
396 	return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(vcpu));
397 }
398 
399 static inline
kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu * vcpu)400 bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu)
401 {
402 	return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu));
403 }
404 
405 static inline
kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu * vcpu)406 u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu)
407 {
408 	unsigned long esr = kvm_vcpu_get_esr(vcpu);
409 
410 	BUG_ON(!esr_fsc_is_permission_fault(esr));
411 	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL));
412 }
413 
kvm_vcpu_abt_issea(const struct kvm_vcpu * vcpu)414 static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
415 {
416 	switch (kvm_vcpu_trap_get_fault(vcpu)) {
417 	case ESR_ELx_FSC_EXTABT:
418 	case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
419 	case ESR_ELx_FSC_SECC:
420 	case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
421 		return true;
422 	default:
423 		return false;
424 	}
425 }
426 
kvm_vcpu_sys_get_rt(struct kvm_vcpu * vcpu)427 static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
428 {
429 	u64 esr = kvm_vcpu_get_esr(vcpu);
430 	return ESR_ELx_SYS64_ISS_RT(esr);
431 }
432 
kvm_is_write_fault(struct kvm_vcpu * vcpu)433 static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
434 {
435 	if (kvm_vcpu_abt_iss1tw(vcpu)) {
436 		/*
437 		 * Only a permission fault on a S1PTW should be
438 		 * considered as a write. Otherwise, page tables baked
439 		 * in a read-only memslot will result in an exception
440 		 * being delivered in the guest.
441 		 *
442 		 * The drawback is that we end-up faulting twice if the
443 		 * guest is using any of HW AF/DB: a translation fault
444 		 * to map the page containing the PT (read only at
445 		 * first), then a permission fault to allow the flags
446 		 * to be set.
447 		 */
448 		return kvm_vcpu_trap_is_permission_fault(vcpu);
449 	}
450 
451 	if (kvm_vcpu_trap_is_iabt(vcpu))
452 		return false;
453 
454 	return kvm_vcpu_dabt_iswrite(vcpu);
455 }
456 
kvm_vcpu_get_mpidr_aff(struct kvm_vcpu * vcpu)457 static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
458 {
459 	return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
460 }
461 
kvm_vcpu_set_be(struct kvm_vcpu * vcpu)462 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
463 {
464 	if (vcpu_mode_is_32bit(vcpu)) {
465 		*vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
466 	} else {
467 		u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
468 		sctlr |= SCTLR_ELx_EE;
469 		vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
470 	}
471 }
472 
kvm_vcpu_is_be(struct kvm_vcpu * vcpu)473 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
474 {
475 	if (vcpu_mode_is_32bit(vcpu))
476 		return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
477 
478 	if (vcpu_mode_priv(vcpu))
479 		return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_EE);
480 	else
481 		return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_EL1_E0E);
482 }
483 
vcpu_data_guest_to_host(struct kvm_vcpu * vcpu,unsigned long data,unsigned int len)484 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
485 						    unsigned long data,
486 						    unsigned int len)
487 {
488 	if (kvm_vcpu_is_be(vcpu)) {
489 		switch (len) {
490 		case 1:
491 			return data & 0xff;
492 		case 2:
493 			return be16_to_cpu(data & 0xffff);
494 		case 4:
495 			return be32_to_cpu(data & 0xffffffff);
496 		default:
497 			return be64_to_cpu(data);
498 		}
499 	} else {
500 		switch (len) {
501 		case 1:
502 			return data & 0xff;
503 		case 2:
504 			return le16_to_cpu(data & 0xffff);
505 		case 4:
506 			return le32_to_cpu(data & 0xffffffff);
507 		default:
508 			return le64_to_cpu(data);
509 		}
510 	}
511 
512 	return data;		/* Leave LE untouched */
513 }
514 
vcpu_data_host_to_guest(struct kvm_vcpu * vcpu,unsigned long data,unsigned int len)515 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
516 						    unsigned long data,
517 						    unsigned int len)
518 {
519 	if (kvm_vcpu_is_be(vcpu)) {
520 		switch (len) {
521 		case 1:
522 			return data & 0xff;
523 		case 2:
524 			return cpu_to_be16(data & 0xffff);
525 		case 4:
526 			return cpu_to_be32(data & 0xffffffff);
527 		default:
528 			return cpu_to_be64(data);
529 		}
530 	} else {
531 		switch (len) {
532 		case 1:
533 			return data & 0xff;
534 		case 2:
535 			return cpu_to_le16(data & 0xffff);
536 		case 4:
537 			return cpu_to_le32(data & 0xffffffff);
538 		default:
539 			return cpu_to_le64(data);
540 		}
541 	}
542 
543 	return data;		/* Leave LE untouched */
544 }
545 
kvm_incr_pc(struct kvm_vcpu * vcpu)546 static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
547 {
548 	WARN_ON(vcpu_get_flag(vcpu, PENDING_EXCEPTION));
549 	vcpu_set_flag(vcpu, INCREMENT_PC);
550 }
551 
552 #define kvm_pend_exception(v, e)					\
553 	do {								\
554 		WARN_ON(vcpu_get_flag((v), INCREMENT_PC));		\
555 		vcpu_set_flag((v), PENDING_EXCEPTION);			\
556 		vcpu_set_flag((v), e);					\
557 	} while (0)
558 
559 #define __build_check_all_or_none(r, bits)				\
560 	BUILD_BUG_ON(((r) & (bits)) && ((r) & (bits)) != (bits))
561 
562 #define __cpacr_to_cptr_clr(clr, set)					\
563 	({								\
564 		u64 cptr = 0;						\
565 									\
566 		if ((set) & CPACR_EL1_FPEN)				\
567 			cptr |= CPTR_EL2_TFP;				\
568 		if ((set) & CPACR_EL1_ZEN)				\
569 			cptr |= CPTR_EL2_TZ;				\
570 		if ((set) & CPACR_EL1_SMEN)				\
571 			cptr |= CPTR_EL2_TSM;				\
572 		if ((clr) & CPACR_EL1_TTA)				\
573 			cptr |= CPTR_EL2_TTA;				\
574 		if ((clr) & CPTR_EL2_TAM)				\
575 			cptr |= CPTR_EL2_TAM;				\
576 		if ((clr) & CPTR_EL2_TCPAC)				\
577 			cptr |= CPTR_EL2_TCPAC;				\
578 									\
579 		cptr;							\
580 	})
581 
582 #define __cpacr_to_cptr_set(clr, set)					\
583 	({								\
584 		u64 cptr = 0;						\
585 									\
586 		if ((clr) & CPACR_EL1_FPEN)				\
587 			cptr |= CPTR_EL2_TFP;				\
588 		if ((clr) & CPACR_EL1_ZEN)				\
589 			cptr |= CPTR_EL2_TZ;				\
590 		if ((clr) & CPACR_EL1_SMEN)				\
591 			cptr |= CPTR_EL2_TSM;				\
592 		if ((set) & CPACR_EL1_TTA)				\
593 			cptr |= CPTR_EL2_TTA;				\
594 		if ((set) & CPTR_EL2_TAM)				\
595 			cptr |= CPTR_EL2_TAM;				\
596 		if ((set) & CPTR_EL2_TCPAC)				\
597 			cptr |= CPTR_EL2_TCPAC;				\
598 									\
599 		cptr;							\
600 	})
601 
602 #define cpacr_clear_set(clr, set)					\
603 	do {								\
604 		BUILD_BUG_ON((set) & CPTR_VHE_EL2_RES0);		\
605 		BUILD_BUG_ON((clr) & CPACR_EL1_E0POE);			\
606 		__build_check_all_or_none((clr), CPACR_EL1_FPEN);	\
607 		__build_check_all_or_none((set), CPACR_EL1_FPEN);	\
608 		__build_check_all_or_none((clr), CPACR_EL1_ZEN);	\
609 		__build_check_all_or_none((set), CPACR_EL1_ZEN);	\
610 		__build_check_all_or_none((clr), CPACR_EL1_SMEN);	\
611 		__build_check_all_or_none((set), CPACR_EL1_SMEN);	\
612 									\
613 		if (has_vhe() || has_hvhe())				\
614 			sysreg_clear_set(cpacr_el1, clr, set);		\
615 		else							\
616 			sysreg_clear_set(cptr_el2,			\
617 					 __cpacr_to_cptr_clr(clr, set),	\
618 					 __cpacr_to_cptr_set(clr, set));\
619 	} while (0)
620 
621 /*
622  * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE
623  * format if E2H isn't set.
624  */
vcpu_sanitised_cptr_el2(const struct kvm_vcpu * vcpu)625 static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu)
626 {
627 	u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2);
628 
629 	if (!vcpu_el2_e2h_is_set(vcpu))
630 		cptr = translate_cptr_el2_to_cpacr_el1(cptr);
631 
632 	return cptr;
633 }
634 
____cptr_xen_trap_enabled(const struct kvm_vcpu * vcpu,unsigned int xen)635 static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu,
636 					     unsigned int xen)
637 {
638 	switch (xen) {
639 	case 0b00:
640 	case 0b10:
641 		return true;
642 	case 0b01:
643 		return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu);
644 	case 0b11:
645 	default:
646 		return false;
647 	}
648 }
649 
650 #define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen)				\
651 	(!vcpu_has_nv(vcpu) ? false :						\
652 	 ____cptr_xen_trap_enabled(vcpu,					\
653 				   SYS_FIELD_GET(CPACR_EL1, xen,		\
654 						 vcpu_sanitised_cptr_el2(vcpu))))
655 
guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu * vcpu)656 static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu)
657 {
658 	return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN);
659 }
660 
guest_hyp_sve_traps_enabled(const struct kvm_vcpu * vcpu)661 static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu)
662 {
663 	return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN);
664 }
665 
vcpu_set_hcrx(struct kvm_vcpu * vcpu)666 static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
667 {
668 	struct kvm *kvm = vcpu->kvm;
669 
670 	if (cpus_have_final_cap(ARM64_HAS_HCX)) {
671 		/*
672 		 * In general, all HCRX_EL2 bits are gated by a feature.
673 		 * The only reason we can set SMPME without checking any
674 		 * feature is that its effects are not directly observable
675 		 * from the guest.
676 		 */
677 		vcpu->arch.hcrx_el2 = HCRX_EL2_SMPME;
678 
679 		if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
680 			vcpu->arch.hcrx_el2 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
681 
682 		if (kvm_has_tcr2(kvm))
683 			vcpu->arch.hcrx_el2 |= HCRX_EL2_TCR2En;
684 
685 		if (kvm_has_fpmr(kvm))
686 			vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;
687 	}
688 }
689 #endif /* __ARM64_KVM_EMULATE_H__ */
690