xref: /linux/arch/arm64/include/asm/kvm_nested.h (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __ARM64_KVM_NESTED_H
3 #define __ARM64_KVM_NESTED_H
4 
5 #include <linux/bitfield.h>
6 #include <linux/kvm_host.h>
7 #include <asm/kvm_emulate.h>
8 #include <asm/kvm_pgtable.h>
9 
10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
11 {
12 	return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
13 		cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
14 		vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
15 }
16 
17 /* Translation helpers from non-VHE EL2 to EL1 */
18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
19 {
20 	return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
21 }
22 
23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
24 {
25 	return TCR_EPD1_MASK |				/* disable TTBR1_EL1 */
26 	       ((tcr & TCR_EL2_DS) ? TCR_DS : 0) |
27 	       ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
28 	       tcr_el2_ps_to_tcr_el1_ips(tcr) |
29 	       (tcr & TCR_EL2_TG0_MASK) |
30 	       (tcr & TCR_EL2_ORGN0_MASK) |
31 	       (tcr & TCR_EL2_IRGN0_MASK) |
32 	       (tcr & TCR_EL2_T0SZ_MASK);
33 }
34 
35 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
36 {
37 	u64 cpacr_el1 = CPACR_EL1_RES1;
38 
39 	if (cptr_el2 & CPTR_EL2_TTA)
40 		cpacr_el1 |= CPACR_EL1_TTA;
41 	if (!(cptr_el2 & CPTR_EL2_TFP))
42 		cpacr_el1 |= CPACR_EL1_FPEN;
43 	if (!(cptr_el2 & CPTR_EL2_TZ))
44 		cpacr_el1 |= CPACR_EL1_ZEN;
45 
46 	cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);
47 
48 	return cpacr_el1;
49 }
50 
51 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
52 {
53 	/* Only preserve the minimal set of bits we support */
54 	val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
55 		SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
56 	val |= SCTLR_EL1_RES1;
57 
58 	return val;
59 }
60 
61 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
62 {
63 	/* Clear the ASID field */
64 	return ttbr0 & ~GENMASK_ULL(63, 48);
65 }
66 
67 extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
68 extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
69 extern int kvm_init_nested(struct kvm *kvm);
70 extern void kvm_destroy_nested(struct kvm *kvm);
71 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
72 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
73 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);
74 
75 union tlbi_info;
76 
77 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
78 				       const union tlbi_info *info,
79 				       void (*)(struct kvm_s2_mmu *,
80 						const union tlbi_info *));
81 extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa,
82 					gpa_t nested_ipa, size_t map_size,
83 					struct kvm_guest_s2_mapping *mapping);
84 extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu,
85 					 gpa_t nipa, size_t size);
86 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
87 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
88 
89 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
90 extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
91 extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
92 
93 extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu);
94 
95 struct kvm_s2_trans {
96 	phys_addr_t output;
97 	unsigned long block_size;
98 	bool writable;
99 	bool readable;
100 	int level;
101 	u32 esr;
102 	u64 desc;
103 };
104 
105 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)
106 {
107 	return trans->output;
108 }
109 
110 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)
111 {
112 	return trans->block_size;
113 }
114 
115 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)
116 {
117 	return trans->esr;
118 }
119 
120 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)
121 {
122 	return trans->readable;
123 }
124 
125 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)
126 {
127 	return trans->writable;
128 }
129 
130 static inline bool kvm_has_xnx(struct kvm *kvm)
131 {
132 	return cpus_have_final_cap(ARM64_HAS_XNX) &&
133 		kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP);
134 }
135 
136 static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans)
137 {
138 	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
139 
140 	if (!kvm_has_xnx(kvm))
141 		xn &= 0b10;
142 
143 	switch (xn) {
144 	case 0b00:
145 	case 0b01:
146 		return true;
147 	default:
148 		return false;
149 	}
150 }
151 
152 static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans)
153 {
154 	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
155 
156 	if (!kvm_has_xnx(kvm))
157 		xn &= 0b10;
158 
159 	switch (xn) {
160 	case 0b00:
161 	case 0b11:
162 		return true;
163 	default:
164 		return false;
165 	}
166 }
167 
168 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
169 			      struct kvm_s2_trans *result);
170 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
171 				    struct kvm_s2_trans *trans);
172 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
173 extern void kvm_nested_s2_wp(struct kvm *kvm);
174 extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa,
175 					size_t unmap_size, bool may_block);
176 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
177 extern void kvm_nested_s2_flush(struct kvm *kvm);
178 
179 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);
180 
181 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)
182 {
183 	struct kvm *kvm = vpcu->kvm;
184 	u8 CRm = sys_reg_CRm(instr);
185 
186 	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
187 	      sys_reg_Op1(instr) == TLBI_Op1_EL1))
188 		return false;
189 
190 	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
191 	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
192 	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
193 		return false;
194 
195 	if (CRm == TLBI_CRm_nROS &&
196 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
197 		return false;
198 
199 	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
200 	     CRm == TLBI_CRm_RNS) &&
201 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
202 		return false;
203 
204 	return true;
205 }
206 
207 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)
208 {
209 	struct kvm *kvm = vpcu->kvm;
210 	u8 CRm = sys_reg_CRm(instr);
211 
212 	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
213 	      sys_reg_Op1(instr) == TLBI_Op1_EL2))
214 		return false;
215 
216 	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
217 	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
218 	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
219 		return false;
220 
221 	if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)
222 		return false;
223 
224 	if (CRm == TLBI_CRm_nROS &&
225 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
226 		return false;
227 
228 	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
229 	     CRm == TLBI_CRm_RNS) &&
230 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
231 		return false;
232 
233 	return true;
234 }
235 
236 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu);
237 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val);
238 
239 #ifdef CONFIG_ARM64_PTR_AUTH
240 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr);
241 #else
242 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr)
243 {
244 	/* We really should never execute this... */
245 	WARN_ON_ONCE(1);
246 	*elr = 0xbad9acc0debadbad;
247 	return false;
248 }
249 #endif
250 
251 #define KVM_NV_GUEST_MAP_SZ	(KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)
252 
253 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)
254 {
255 	return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);
256 }
257 
258 /* Adjust alignment for the contiguous bit as per StageOA() */
259 #define contiguous_bit_shift(d, wi, l)					\
260 	({								\
261 		u8 shift = 0;						\
262 									\
263 		if ((d) & PTE_CONT) {					\
264 			switch (BIT((wi)->pgshift)) {			\
265 			case SZ_4K:					\
266 				shift = 4;				\
267 				break;					\
268 			case SZ_16K:					\
269 				shift = (l) == 2 ? 5 : 7;		\
270 				break;					\
271 			case SZ_64K:					\
272 				shift = 5;				\
273 				break;					\
274 			}						\
275 		}							\
276 									\
277 		shift;							\
278 	})
279 
280 static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
281 {
282 	u64 base, tg, num, scale;
283 	int shift;
284 
285 	tg	= FIELD_GET(GENMASK(47, 46), val);
286 
287 	switch(tg) {
288 	case 1:
289 		shift = 12;
290 		break;
291 	case 2:
292 		shift = 14;
293 		break;
294 	case 3:
295 	default:		/* IMPDEF: handle tg==0 as 64k */
296 		shift = 16;
297 		break;
298 	}
299 
300 	base	= (val & GENMASK(36, 0)) << shift;
301 
302 	/*
303 	 * We only deal with at most 48bit VA/IPA, so 48 is where we
304 	 * sign-extend from. Should we support FEAT_L{VP}A* at some point,
305 	 * this will need to be revisited.
306 	 */
307 	base	= (u64)sign_extend64(base, 48);
308 
309 	if (asid)
310 		*asid = FIELD_GET(TLBIR_ASID_MASK, val);
311 
312 	scale	= FIELD_GET(GENMASK(45, 44), val);
313 	num	= FIELD_GET(GENMASK(43, 39), val);
314 	*range	= __TLBI_RANGE_PAGES(num, scale) << shift;
315 
316 	/* Cap the range to the correct half of the address space */
317 	if (!(base & BIT(48)))
318 		*range = min(*range, (BIT(48) - base));
319 	else
320 		*range = min(*range, ~base + 1);
321 
322 	return base;
323 }
324 
325 static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit)
326 {
327 	switch (ps) {
328 	case 0: return 32;
329 	case 1: return 36;
330 	case 2: return 40;
331 	case 3: return 42;
332 	case 4: return 44;
333 	case 5: return 48;
334 	case 6: if (pa52bit)
335 			return 52;
336 		fallthrough;
337 	default:
338 		return 48;
339 	}
340 }
341 
342 enum trans_regime {
343 	TR_EL10,
344 	TR_EL20,
345 	TR_EL2,
346 };
347 
348 struct s1_walk_info;
349 
350 struct s1_walk_context {
351 	struct s1_walk_info	*wi;
352 	u64			table_ipa;
353 	int			level;
354 };
355 
356 struct s1_walk_filter {
357 	int	(*fn)(struct s1_walk_context *, void *);
358 	void	*priv;
359 };
360 
361 struct s1_walk_info {
362 	struct s1_walk_filter	*filter;
363 	u64	     		baddr;
364 	enum trans_regime	regime;
365 	unsigned int		max_oa_bits;
366 	unsigned int		pgshift;
367 	unsigned int		txsz;
368 	int 	     		sl;
369 	u8			sh;
370 	bool			as_el0;
371 	bool	     		hpd;
372 	bool			e0poe;
373 	bool			poe;
374 	bool			pan;
375 	bool	     		be;
376 	bool	     		s2;
377 	bool			pa52bit;
378 	bool			ha;
379 };
380 
381 struct s1_walk_result {
382 	union {
383 		struct {
384 			u64	desc;
385 			u64	pa;
386 			s8	level;
387 			u8	APTable;
388 			bool	nG;
389 			u16	asid;
390 			bool	UXNTable;
391 			bool	PXNTable;
392 			bool	uwxn;
393 			bool	uov;
394 			bool	ur;
395 			bool	uw;
396 			bool	ux;
397 			bool	pwxn;
398 			bool	pov;
399 			bool	pr;
400 			bool	pw;
401 			bool	px;
402 		};
403 		struct {
404 			u8	fst;
405 			bool	ptw;
406 			bool	s2;
407 		};
408 	};
409 	bool	failed;
410 };
411 
412 #define S1_MMU_DISABLED		(-127)
413 
414 static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
415 {
416 	wr->fst		= fst;
417 	wr->ptw		= s1ptw;
418 	wr->s2		= s1ptw;
419 	wr->failed	= true;
420 }
421 
422 static inline bool s1_walk_translated(struct s1_walk_result *wr)
423 {
424 	return wr->level != S1_MMU_DISABLED;
425 }
426 
427 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
428 		       struct s1_walk_result *wr, u64 va);
429 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa,
430 			     int *level);
431 
432 /* VNCR management */
433 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
434 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
435 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);
436 
437 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime);
438 
439 #define vncr_fixmap(c)						\
440 	({							\
441 		u32 __c = (c);					\
442 		BUG_ON(__c >= NR_CPUS);				\
443 		(FIX_VNCR - __c);				\
444 	})
445 
446 int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new);
447 
448 #endif /* __ARM64_KVM_NESTED_H */
449