xref: /linux/arch/arm64/kvm/nested.c (revision 954a209f431c06b62718a49b403bd4c549f0d6fb)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4  * Author: Jintack Lim <jintack.lim@linaro.org>
5  */
6 
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10 
11 #include <asm/kvm_arm.h>
12 #include <asm/kvm_emulate.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/kvm_nested.h>
15 #include <asm/sysreg.h>
16 
17 #include "sys_regs.h"
18 
19 /* Protection against the sysreg repainting madness... */
20 #define NV_FTR(r, f)		ID_AA64##r##_EL1_##f
21 
22 /*
23  * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
24  * memory usage and potential number of different sets of S2 PTs in
25  * the guests. Running out of S2 MMUs only affects performance (we
26  * will invalidate them more often).
27  */
28 #define S2_MMU_PER_VCPU		2
29 
kvm_init_nested(struct kvm * kvm)30 void kvm_init_nested(struct kvm *kvm)
31 {
32 	kvm->arch.nested_mmus = NULL;
33 	kvm->arch.nested_mmus_size = 0;
34 }
35 
init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
37 {
38 	/*
39 	 * We only initialise the IPA range on the canonical MMU, which
40 	 * defines the contract between KVM and userspace on where the
41 	 * "hardware" is in the IPA space. This affects the validity of MMIO
42 	 * exits forwarded to userspace, for example.
43 	 *
44 	 * For nested S2s, we use the PARange as exposed to the guest, as it
45 	 * is allowed to use it at will to expose whatever memory map it
46 	 * wants to its own guests as it would be on real HW.
47 	 */
48 	return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
49 }
50 
kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
52 {
53 	struct kvm *kvm = vcpu->kvm;
54 	struct kvm_s2_mmu *tmp;
55 	int num_mmus, ret = 0;
56 
57 	/*
58 	 * Let's treat memory allocation failures as benign: If we fail to
59 	 * allocate anything, return an error and keep the allocated array
60 	 * alive. Userspace may try to recover by intializing the vcpu
61 	 * again, and there is no reason to affect the whole VM for this.
62 	 */
63 	num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
64 	tmp = kvrealloc(kvm->arch.nested_mmus,
65 			size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus),
66 			GFP_KERNEL_ACCOUNT | __GFP_ZERO);
67 	if (!tmp)
68 		return -ENOMEM;
69 
70 	swap(kvm->arch.nested_mmus, tmp);
71 
72 	/*
73 	 * If we went through a realocation, adjust the MMU back-pointers in
74 	 * the previously initialised kvm_pgtable structures.
75 	 */
76 	if (kvm->arch.nested_mmus != tmp)
77 		for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
78 			kvm->arch.nested_mmus[i].pgt->mmu = &kvm->arch.nested_mmus[i];
79 
80 	for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
81 		ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
82 
83 	if (ret) {
84 		for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
85 			kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
86 
87 		return ret;
88 	}
89 
90 	kvm->arch.nested_mmus_size = num_mmus;
91 
92 	return 0;
93 }
94 
95 struct s2_walk_info {
96 	int	     (*read_desc)(phys_addr_t pa, u64 *desc, void *data);
97 	void	     *data;
98 	u64	     baddr;
99 	unsigned int max_oa_bits;
100 	unsigned int pgshift;
101 	unsigned int sl;
102 	unsigned int t0sz;
103 	bool	     be;
104 };
105 
compute_fsc(int level,u32 fsc)106 static u32 compute_fsc(int level, u32 fsc)
107 {
108 	return fsc | (level & 0x3);
109 }
110 
esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)111 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
112 {
113 	u32 esr;
114 
115 	esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
116 	esr |= compute_fsc(level, fsc);
117 	return esr;
118 }
119 
get_ia_size(struct s2_walk_info * wi)120 static int get_ia_size(struct s2_walk_info *wi)
121 {
122 	return 64 - wi->t0sz;
123 }
124 
check_base_s2_limits(struct s2_walk_info * wi,int level,int input_size,int stride)125 static int check_base_s2_limits(struct s2_walk_info *wi,
126 				int level, int input_size, int stride)
127 {
128 	int start_size, ia_size;
129 
130 	ia_size = get_ia_size(wi);
131 
132 	/* Check translation limits */
133 	switch (BIT(wi->pgshift)) {
134 	case SZ_64K:
135 		if (level == 0 || (level == 1 && ia_size <= 42))
136 			return -EFAULT;
137 		break;
138 	case SZ_16K:
139 		if (level == 0 || (level == 1 && ia_size <= 40))
140 			return -EFAULT;
141 		break;
142 	case SZ_4K:
143 		if (level < 0 || (level == 0 && ia_size <= 42))
144 			return -EFAULT;
145 		break;
146 	}
147 
148 	/* Check input size limits */
149 	if (input_size > ia_size)
150 		return -EFAULT;
151 
152 	/* Check number of entries in starting level table */
153 	start_size = input_size - ((3 - level) * stride + wi->pgshift);
154 	if (start_size < 1 || start_size > stride + 4)
155 		return -EFAULT;
156 
157 	return 0;
158 }
159 
160 /* Check if output is within boundaries */
check_output_size(struct s2_walk_info * wi,phys_addr_t output)161 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
162 {
163 	unsigned int output_size = wi->max_oa_bits;
164 
165 	if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
166 		return -1;
167 
168 	return 0;
169 }
170 
171 /*
172  * This is essentially a C-version of the pseudo code from the ARM ARM
173  * AArch64.TranslationTableWalk  function.  I strongly recommend looking at
174  * that pseudocode in trying to understand this.
175  *
176  * Must be called with the kvm->srcu read lock held
177  */
walk_nested_s2_pgd(phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)178 static int walk_nested_s2_pgd(phys_addr_t ipa,
179 			      struct s2_walk_info *wi, struct kvm_s2_trans *out)
180 {
181 	int first_block_level, level, stride, input_size, base_lower_bound;
182 	phys_addr_t base_addr;
183 	unsigned int addr_top, addr_bottom;
184 	u64 desc;  /* page table entry */
185 	int ret;
186 	phys_addr_t paddr;
187 
188 	switch (BIT(wi->pgshift)) {
189 	default:
190 	case SZ_64K:
191 	case SZ_16K:
192 		level = 3 - wi->sl;
193 		first_block_level = 2;
194 		break;
195 	case SZ_4K:
196 		level = 2 - wi->sl;
197 		first_block_level = 1;
198 		break;
199 	}
200 
201 	stride = wi->pgshift - 3;
202 	input_size = get_ia_size(wi);
203 	if (input_size > 48 || input_size < 25)
204 		return -EFAULT;
205 
206 	ret = check_base_s2_limits(wi, level, input_size, stride);
207 	if (WARN_ON(ret))
208 		return ret;
209 
210 	base_lower_bound = 3 + input_size - ((3 - level) * stride +
211 			   wi->pgshift);
212 	base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
213 
214 	if (check_output_size(wi, base_addr)) {
215 		out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
216 		return 1;
217 	}
218 
219 	addr_top = input_size - 1;
220 
221 	while (1) {
222 		phys_addr_t index;
223 
224 		addr_bottom = (3 - level) * stride + wi->pgshift;
225 		index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
226 			>> (addr_bottom - 3);
227 
228 		paddr = base_addr | index;
229 		ret = wi->read_desc(paddr, &desc, wi->data);
230 		if (ret < 0)
231 			return ret;
232 
233 		/*
234 		 * Handle reversedescriptors if endianness differs between the
235 		 * host and the guest hypervisor.
236 		 */
237 		if (wi->be)
238 			desc = be64_to_cpu((__force __be64)desc);
239 		else
240 			desc = le64_to_cpu((__force __le64)desc);
241 
242 		/* Check for valid descriptor at this point */
243 		if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) {
244 			out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
245 			out->desc = desc;
246 			return 1;
247 		}
248 
249 		/* We're at the final level or block translation level */
250 		if ((desc & 3) == 1 || level == 3)
251 			break;
252 
253 		if (check_output_size(wi, desc)) {
254 			out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
255 			out->desc = desc;
256 			return 1;
257 		}
258 
259 		base_addr = desc & GENMASK_ULL(47, wi->pgshift);
260 
261 		level += 1;
262 		addr_top = addr_bottom - 1;
263 	}
264 
265 	if (level < first_block_level) {
266 		out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
267 		out->desc = desc;
268 		return 1;
269 	}
270 
271 	if (check_output_size(wi, desc)) {
272 		out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
273 		out->desc = desc;
274 		return 1;
275 	}
276 
277 	if (!(desc & BIT(10))) {
278 		out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
279 		out->desc = desc;
280 		return 1;
281 	}
282 
283 	addr_bottom += contiguous_bit_shift(desc, wi, level);
284 
285 	/* Calculate and return the result */
286 	paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
287 		(ipa & GENMASK_ULL(addr_bottom - 1, 0));
288 	out->output = paddr;
289 	out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
290 	out->readable = desc & (0b01 << 6);
291 	out->writable = desc & (0b10 << 6);
292 	out->level = level;
293 	out->desc = desc;
294 	return 0;
295 }
296 
read_guest_s2_desc(phys_addr_t pa,u64 * desc,void * data)297 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data)
298 {
299 	struct kvm_vcpu *vcpu = data;
300 
301 	return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc));
302 }
303 
vtcr_to_walk_info(u64 vtcr,struct s2_walk_info * wi)304 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
305 {
306 	wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
307 
308 	switch (vtcr & VTCR_EL2_TG0_MASK) {
309 	case VTCR_EL2_TG0_4K:
310 		wi->pgshift = 12;	 break;
311 	case VTCR_EL2_TG0_16K:
312 		wi->pgshift = 14;	 break;
313 	case VTCR_EL2_TG0_64K:
314 	default:	    /* IMPDEF: treat any other value as 64k */
315 		wi->pgshift = 16;	 break;
316 	}
317 
318 	wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
319 	/* Global limit for now, should eventually be per-VM */
320 	wi->max_oa_bits = min(get_kvm_ipa_limit(),
321 			      ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr)));
322 }
323 
kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)324 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
325 		       struct kvm_s2_trans *result)
326 {
327 	u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
328 	struct s2_walk_info wi;
329 	int ret;
330 
331 	result->esr = 0;
332 
333 	if (!vcpu_has_nv(vcpu))
334 		return 0;
335 
336 	wi.read_desc = read_guest_s2_desc;
337 	wi.data = vcpu;
338 	wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
339 
340 	vtcr_to_walk_info(vtcr, &wi);
341 
342 	wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
343 
344 	ret = walk_nested_s2_pgd(gipa, &wi, result);
345 	if (ret)
346 		result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
347 
348 	return ret;
349 }
350 
ttl_to_size(u8 ttl)351 static unsigned int ttl_to_size(u8 ttl)
352 {
353 	int level = ttl & 3;
354 	int gran = (ttl >> 2) & 3;
355 	unsigned int max_size = 0;
356 
357 	switch (gran) {
358 	case TLBI_TTL_TG_4K:
359 		switch (level) {
360 		case 0:
361 			break;
362 		case 1:
363 			max_size = SZ_1G;
364 			break;
365 		case 2:
366 			max_size = SZ_2M;
367 			break;
368 		case 3:
369 			max_size = SZ_4K;
370 			break;
371 		}
372 		break;
373 	case TLBI_TTL_TG_16K:
374 		switch (level) {
375 		case 0:
376 		case 1:
377 			break;
378 		case 2:
379 			max_size = SZ_32M;
380 			break;
381 		case 3:
382 			max_size = SZ_16K;
383 			break;
384 		}
385 		break;
386 	case TLBI_TTL_TG_64K:
387 		switch (level) {
388 		case 0:
389 		case 1:
390 			/* No 52bit IPA support */
391 			break;
392 		case 2:
393 			max_size = SZ_512M;
394 			break;
395 		case 3:
396 			max_size = SZ_64K;
397 			break;
398 		}
399 		break;
400 	default:			/* No size information */
401 		break;
402 	}
403 
404 	return max_size;
405 }
406 
407 /*
408  * Compute the equivalent of the TTL field by parsing the shadow PT.  The
409  * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
410  * retrieved from first entry carrying the level as a tag.
411  */
get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)412 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
413 {
414 	u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
415 	kvm_pte_t pte;
416 	u8 ttl, level;
417 
418 	lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
419 
420 	switch (vtcr & VTCR_EL2_TG0_MASK) {
421 	case VTCR_EL2_TG0_4K:
422 		ttl = (TLBI_TTL_TG_4K << 2);
423 		break;
424 	case VTCR_EL2_TG0_16K:
425 		ttl = (TLBI_TTL_TG_16K << 2);
426 		break;
427 	case VTCR_EL2_TG0_64K:
428 	default:	    /* IMPDEF: treat any other value as 64k */
429 		ttl = (TLBI_TTL_TG_64K << 2);
430 		break;
431 	}
432 
433 	tmp = addr;
434 
435 again:
436 	/* Iteratively compute the block sizes for a particular granule size */
437 	switch (vtcr & VTCR_EL2_TG0_MASK) {
438 	case VTCR_EL2_TG0_4K:
439 		if	(sz < SZ_4K)	sz = SZ_4K;
440 		else if (sz < SZ_2M)	sz = SZ_2M;
441 		else if (sz < SZ_1G)	sz = SZ_1G;
442 		else			sz = 0;
443 		break;
444 	case VTCR_EL2_TG0_16K:
445 		if	(sz < SZ_16K)	sz = SZ_16K;
446 		else if (sz < SZ_32M)	sz = SZ_32M;
447 		else			sz = 0;
448 		break;
449 	case VTCR_EL2_TG0_64K:
450 	default:	    /* IMPDEF: treat any other value as 64k */
451 		if	(sz < SZ_64K)	sz = SZ_64K;
452 		else if (sz < SZ_512M)	sz = SZ_512M;
453 		else			sz = 0;
454 		break;
455 	}
456 
457 	if (sz == 0)
458 		return 0;
459 
460 	tmp &= ~(sz - 1);
461 	if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
462 		goto again;
463 	if (!(pte & PTE_VALID))
464 		goto again;
465 	level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
466 	if (!level)
467 		goto again;
468 
469 	ttl |= level;
470 
471 	/*
472 	 * We now have found some level information in the shadow S2. Check
473 	 * that the resulting range is actually including the original IPA.
474 	 */
475 	sz = ttl_to_size(ttl);
476 	if (addr < (tmp + sz))
477 		return ttl;
478 
479 	return 0;
480 }
481 
compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)482 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
483 {
484 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
485 	unsigned long max_size;
486 	u8 ttl;
487 
488 	ttl = FIELD_GET(TLBI_TTL_MASK, val);
489 
490 	if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
491 		/* No TTL, check the shadow S2 for a hint */
492 		u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
493 		ttl = get_guest_mapping_ttl(mmu, addr);
494 	}
495 
496 	max_size = ttl_to_size(ttl);
497 
498 	if (!max_size) {
499 		/* Compute the maximum extent of the invalidation */
500 		switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) {
501 		case VTCR_EL2_TG0_4K:
502 			max_size = SZ_1G;
503 			break;
504 		case VTCR_EL2_TG0_16K:
505 			max_size = SZ_32M;
506 			break;
507 		case VTCR_EL2_TG0_64K:
508 		default:    /* IMPDEF: treat any other value as 64k */
509 			/*
510 			 * No, we do not support 52bit IPA in nested yet. Once
511 			 * we do, this should be 4TB.
512 			 */
513 			max_size = SZ_512M;
514 			break;
515 		}
516 	}
517 
518 	WARN_ON(!max_size);
519 	return max_size;
520 }
521 
522 /*
523  * We can have multiple *different* MMU contexts with the same VMID:
524  *
525  * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
526  *
527  * - Multiple vcpus using private S2s (huh huh...), hence differing by the
528  *   VBBTR_EL2.BADDR address
529  *
530  * - A combination of the above...
531  *
532  * We can always identify which MMU context to pick at run-time.  However,
533  * TLB invalidation involving a VMID must take action on all the TLBs using
534  * this particular VMID. This translates into applying the same invalidation
535  * operation to all the contexts that are using this VMID. Moar phun!
536  */
kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))537 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
538 				const union tlbi_info *info,
539 				void (*tlbi_callback)(struct kvm_s2_mmu *,
540 						      const union tlbi_info *))
541 {
542 	write_lock(&kvm->mmu_lock);
543 
544 	for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
545 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
546 
547 		if (!kvm_s2_mmu_valid(mmu))
548 			continue;
549 
550 		if (vmid == get_vmid(mmu->tlb_vttbr))
551 			tlbi_callback(mmu, info);
552 	}
553 
554 	write_unlock(&kvm->mmu_lock);
555 }
556 
lookup_s2_mmu(struct kvm_vcpu * vcpu)557 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
558 {
559 	struct kvm *kvm = vcpu->kvm;
560 	bool nested_stage2_enabled;
561 	u64 vttbr, vtcr, hcr;
562 
563 	lockdep_assert_held_write(&kvm->mmu_lock);
564 
565 	vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
566 	vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
567 	hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
568 
569 	nested_stage2_enabled = hcr & HCR_VM;
570 
571 	/* Don't consider the CnP bit for the vttbr match */
572 	vttbr &= ~VTTBR_CNP_BIT;
573 
574 	/*
575 	 * Two possibilities when looking up a S2 MMU context:
576 	 *
577 	 * - either S2 is enabled in the guest, and we need a context that is
578 	 *   S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
579 	 *   which makes it safe from a TLB conflict perspective (a broken
580 	 *   guest won't be able to generate them),
581 	 *
582 	 * - or S2 is disabled, and we need a context that is S2-disabled
583 	 *   and matches the VMID only, as all TLBs are tagged by VMID even
584 	 *   if S2 translation is disabled.
585 	 */
586 	for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
587 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
588 
589 		if (!kvm_s2_mmu_valid(mmu))
590 			continue;
591 
592 		if (nested_stage2_enabled &&
593 		    mmu->nested_stage2_enabled &&
594 		    vttbr == mmu->tlb_vttbr &&
595 		    vtcr == mmu->tlb_vtcr)
596 			return mmu;
597 
598 		if (!nested_stage2_enabled &&
599 		    !mmu->nested_stage2_enabled &&
600 		    get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
601 			return mmu;
602 	}
603 	return NULL;
604 }
605 
get_s2_mmu_nested(struct kvm_vcpu * vcpu)606 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
607 {
608 	struct kvm *kvm = vcpu->kvm;
609 	struct kvm_s2_mmu *s2_mmu;
610 	int i;
611 
612 	lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
613 
614 	s2_mmu = lookup_s2_mmu(vcpu);
615 	if (s2_mmu)
616 		goto out;
617 
618 	/*
619 	 * Make sure we don't always search from the same point, or we
620 	 * will always reuse a potentially active context, leaving
621 	 * free contexts unused.
622 	 */
623 	for (i = kvm->arch.nested_mmus_next;
624 	     i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
625 	     i++) {
626 		s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
627 
628 		if (atomic_read(&s2_mmu->refcnt) == 0)
629 			break;
630 	}
631 	BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
632 
633 	/* Set the scene for the next search */
634 	kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
635 
636 	/* Make sure we don't forget to do the laundry */
637 	if (kvm_s2_mmu_valid(s2_mmu))
638 		s2_mmu->pending_unmap = true;
639 
640 	/*
641 	 * The virtual VMID (modulo CnP) will be used as a key when matching
642 	 * an existing kvm_s2_mmu.
643 	 *
644 	 * We cache VTCR at allocation time, once and for all. It'd be great
645 	 * if the guest didn't screw that one up, as this is not very
646 	 * forgiving...
647 	 */
648 	s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
649 	s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
650 	s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
651 
652 out:
653 	atomic_inc(&s2_mmu->refcnt);
654 
655 	/*
656 	 * Set the vCPU request to perform an unmap, even if the pending unmap
657 	 * originates from another vCPU. This guarantees that the MMU has been
658 	 * completely unmapped before any vCPU actually uses it, and allows
659 	 * multiple vCPUs to lend a hand with completing the unmap.
660 	 */
661 	if (s2_mmu->pending_unmap)
662 		kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu);
663 
664 	return s2_mmu;
665 }
666 
kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)667 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
668 {
669 	/* CnP being set denotes an invalid entry */
670 	mmu->tlb_vttbr = VTTBR_CNP_BIT;
671 	mmu->nested_stage2_enabled = false;
672 	atomic_set(&mmu->refcnt, 0);
673 }
674 
kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)675 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
676 {
677 	/*
678 	 * The vCPU kept its reference on the MMU after the last put, keep
679 	 * rolling with it.
680 	 */
681 	if (vcpu->arch.hw_mmu)
682 		return;
683 
684 	if (is_hyp_ctxt(vcpu)) {
685 		vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
686 	} else {
687 		write_lock(&vcpu->kvm->mmu_lock);
688 		vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
689 		write_unlock(&vcpu->kvm->mmu_lock);
690 	}
691 }
692 
kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)693 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
694 {
695 	/*
696 	 * Keep a reference on the associated stage-2 MMU if the vCPU is
697 	 * scheduling out and not in WFI emulation, suggesting it is likely to
698 	 * reuse the MMU sometime soon.
699 	 */
700 	if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI))
701 		return;
702 
703 	if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
704 		atomic_dec(&vcpu->arch.hw_mmu->refcnt);
705 
706 	vcpu->arch.hw_mmu = NULL;
707 }
708 
709 /*
710  * Returns non-zero if permission fault is handled by injecting it to the next
711  * level hypervisor.
712  */
kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)713 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
714 {
715 	bool forward_fault = false;
716 
717 	trans->esr = 0;
718 
719 	if (!kvm_vcpu_trap_is_permission_fault(vcpu))
720 		return 0;
721 
722 	if (kvm_vcpu_trap_is_iabt(vcpu)) {
723 		forward_fault = !kvm_s2_trans_executable(trans);
724 	} else {
725 		bool write_fault = kvm_is_write_fault(vcpu);
726 
727 		forward_fault = ((write_fault && !trans->writable) ||
728 				 (!write_fault && !trans->readable));
729 	}
730 
731 	if (forward_fault)
732 		trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
733 
734 	return forward_fault;
735 }
736 
kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)737 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
738 {
739 	vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
740 	vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
741 
742 	return kvm_inject_nested_sync(vcpu, esr_el2);
743 }
744 
kvm_nested_s2_wp(struct kvm * kvm)745 void kvm_nested_s2_wp(struct kvm *kvm)
746 {
747 	int i;
748 
749 	lockdep_assert_held_write(&kvm->mmu_lock);
750 
751 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
752 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
753 
754 		if (kvm_s2_mmu_valid(mmu))
755 			kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
756 	}
757 }
758 
kvm_nested_s2_unmap(struct kvm * kvm,bool may_block)759 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
760 {
761 	int i;
762 
763 	lockdep_assert_held_write(&kvm->mmu_lock);
764 
765 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
766 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
767 
768 		if (kvm_s2_mmu_valid(mmu))
769 			kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
770 	}
771 }
772 
kvm_nested_s2_flush(struct kvm * kvm)773 void kvm_nested_s2_flush(struct kvm *kvm)
774 {
775 	int i;
776 
777 	lockdep_assert_held_write(&kvm->mmu_lock);
778 
779 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
780 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
781 
782 		if (kvm_s2_mmu_valid(mmu))
783 			kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
784 	}
785 }
786 
kvm_arch_flush_shadow_all(struct kvm * kvm)787 void kvm_arch_flush_shadow_all(struct kvm *kvm)
788 {
789 	int i;
790 
791 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
792 		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
793 
794 		if (!WARN_ON(atomic_read(&mmu->refcnt)))
795 			kvm_free_stage2_pgd(mmu);
796 	}
797 	kvfree(kvm->arch.nested_mmus);
798 	kvm->arch.nested_mmus = NULL;
799 	kvm->arch.nested_mmus_size = 0;
800 	kvm_uninit_stage2_mmu(kvm);
801 }
802 
803 /*
804  * Our emulated CPU doesn't support all the possible features. For the
805  * sake of simplicity (and probably mental sanity), wipe out a number
806  * of feature bits we don't intend to support for the time being.
807  * This list should get updated as new features get added to the NV
808  * support, and new extension to the architecture.
809  */
limit_nv_id_regs(struct kvm * kvm)810 static void limit_nv_id_regs(struct kvm *kvm)
811 {
812 	u64 val, tmp;
813 
814 	/* Support everything but TME */
815 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1);
816 	val &= ~NV_FTR(ISAR0, TME);
817 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1, val);
818 
819 	/* Support everything but Spec Invalidation and LS64 */
820 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1);
821 	val &= ~(NV_FTR(ISAR1, LS64)	|
822 		 NV_FTR(ISAR1, SPECRES));
823 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val);
824 
825 	/* No AMU, MPAM, S-EL2, or RAS */
826 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1);
827 	val &= ~(GENMASK_ULL(55, 52)	|
828 		 NV_FTR(PFR0, AMU)	|
829 		 NV_FTR(PFR0, MPAM)	|
830 		 NV_FTR(PFR0, SEL2)	|
831 		 NV_FTR(PFR0, RAS)	|
832 		 NV_FTR(PFR0, EL3)	|
833 		 NV_FTR(PFR0, EL2)	|
834 		 NV_FTR(PFR0, EL1)	|
835 		 NV_FTR(PFR0, EL0));
836 	/* 64bit only at any EL */
837 	val |= FIELD_PREP(NV_FTR(PFR0, EL0), 0b0001);
838 	val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001);
839 	val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001);
840 	val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001);
841 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val);
842 
843 	/* Only support BTI, SSBS, CSV2_frac */
844 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1);
845 	val &= (NV_FTR(PFR1, BT)	|
846 		NV_FTR(PFR1, SSBS)	|
847 		NV_FTR(PFR1, CSV2_frac));
848 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1, val);
849 
850 	/* Hide ECV, ExS, Secure Memory */
851 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
852 	val &= ~(NV_FTR(MMFR0, ECV)		|
853 		 NV_FTR(MMFR0, EXS)		|
854 		 NV_FTR(MMFR0, TGRAN4_2)	|
855 		 NV_FTR(MMFR0, TGRAN16_2)	|
856 		 NV_FTR(MMFR0, TGRAN64_2)	|
857 		 NV_FTR(MMFR0, SNSMEM));
858 
859 	/* Disallow unsupported S2 page sizes */
860 	switch (PAGE_SIZE) {
861 	case SZ_64K:
862 		val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001);
863 		fallthrough;
864 	case SZ_16K:
865 		val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001);
866 		fallthrough;
867 	case SZ_4K:
868 		/* Support everything */
869 		break;
870 	}
871 	/*
872 	 * Since we can't support a guest S2 page size smaller than
873 	 * the host's own page size (due to KVM only populating its
874 	 * own S2 using the kernel's page size), advertise the
875 	 * limitation using FEAT_GTG.
876 	 */
877 	switch (PAGE_SIZE) {
878 	case SZ_4K:
879 		val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010);
880 		fallthrough;
881 	case SZ_16K:
882 		val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010);
883 		fallthrough;
884 	case SZ_64K:
885 		val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010);
886 		break;
887 	}
888 	/* Cap PARange to 48bits */
889 	tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val);
890 	if (tmp > 0b0101) {
891 		val &= ~NV_FTR(MMFR0, PARANGE);
892 		val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101);
893 	}
894 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1, val);
895 
896 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1);
897 	val &= (NV_FTR(MMFR1, HCX)	|
898 		NV_FTR(MMFR1, PAN)	|
899 		NV_FTR(MMFR1, LO)	|
900 		NV_FTR(MMFR1, HPDS)	|
901 		NV_FTR(MMFR1, VH)	|
902 		NV_FTR(MMFR1, VMIDBits));
903 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1, val);
904 
905 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1);
906 	val &= ~(NV_FTR(MMFR2, BBM)	|
907 		 NV_FTR(MMFR2, TTL)	|
908 		 GENMASK_ULL(47, 44)	|
909 		 NV_FTR(MMFR2, ST)	|
910 		 NV_FTR(MMFR2, CCIDX)	|
911 		 NV_FTR(MMFR2, VARange));
912 
913 	/* Force TTL support */
914 	val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001);
915 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1, val);
916 
917 	val = 0;
918 	if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1))
919 		val |= FIELD_PREP(NV_FTR(MMFR4, E2H0),
920 				  ID_AA64MMFR4_EL1_E2H0_NI_NV1);
921 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR4_EL1, val);
922 
923 	/* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */
924 	val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1);
925 	val &= (NV_FTR(DFR0, PMUVer)	|
926 		NV_FTR(DFR0, WRPs)	|
927 		NV_FTR(DFR0, BRPs)	|
928 		NV_FTR(DFR0, DebugVer)	|
929 		NV_FTR(DFR0, HPMN0));
930 
931 	/* Cap Debug to ARMv8.1 */
932 	tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val);
933 	if (tmp > 0b0111) {
934 		val &= ~NV_FTR(DFR0, DebugVer);
935 		val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111);
936 	}
937 	kvm_set_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1, val);
938 }
939 
kvm_vcpu_apply_reg_masks(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr,u64 v)940 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu,
941 			     enum vcpu_sysreg sr, u64 v)
942 {
943 	struct kvm_sysreg_masks *masks;
944 
945 	masks = vcpu->kvm->arch.sysreg_masks;
946 
947 	if (masks) {
948 		sr -= __SANITISED_REG_START__;
949 
950 		v &= ~masks->mask[sr].res0;
951 		v |= masks->mask[sr].res1;
952 	}
953 
954 	return v;
955 }
956 
set_sysreg_masks(struct kvm * kvm,int sr,u64 res0,u64 res1)957 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1)
958 {
959 	int i = sr - __SANITISED_REG_START__;
960 
961 	BUILD_BUG_ON(!__builtin_constant_p(sr));
962 	BUILD_BUG_ON(sr < __SANITISED_REG_START__);
963 	BUILD_BUG_ON(sr >= NR_SYS_REGS);
964 
965 	kvm->arch.sysreg_masks->mask[i].res0 = res0;
966 	kvm->arch.sysreg_masks->mask[i].res1 = res1;
967 }
968 
kvm_init_nv_sysregs(struct kvm_vcpu * vcpu)969 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
970 {
971 	struct kvm *kvm = vcpu->kvm;
972 	u64 res0, res1;
973 
974 	lockdep_assert_held(&kvm->arch.config_lock);
975 
976 	if (kvm->arch.sysreg_masks)
977 		goto out;
978 
979 	kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)),
980 					 GFP_KERNEL_ACCOUNT);
981 	if (!kvm->arch.sysreg_masks)
982 		return -ENOMEM;
983 
984 	limit_nv_id_regs(kvm);
985 
986 	/* VTTBR_EL2 */
987 	res0 = res1 = 0;
988 	if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
989 		res0 |= GENMASK(63, 56);
990 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
991 		res0 |= VTTBR_CNP_BIT;
992 	set_sysreg_masks(kvm, VTTBR_EL2, res0, res1);
993 
994 	/* VTCR_EL2 */
995 	res0 = GENMASK(63, 32) | GENMASK(30, 20);
996 	res1 = BIT(31);
997 	set_sysreg_masks(kvm, VTCR_EL2, res0, res1);
998 
999 	/* VMPIDR_EL2 */
1000 	res0 = GENMASK(63, 40) | GENMASK(30, 24);
1001 	res1 = BIT(31);
1002 	set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1);
1003 
1004 	/* HCR_EL2 */
1005 	res0 = BIT(48);
1006 	res1 = HCR_RW;
1007 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP))
1008 		res0 |= GENMASK(63, 59);
1009 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2))
1010 		res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA);
1011 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS))
1012 		res0 |= (HCR_TTLBIS | HCR_TTLBOS);
1013 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
1014 	    !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
1015 		res0 |= HCR_ENSCXT;
1016 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP))
1017 		res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4);
1018 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
1019 		res0 |= HCR_AMVOFFEN;
1020 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1))
1021 		res0 |= HCR_FIEN;
1022 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP))
1023 		res0 |= HCR_FWB;
1024 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2))
1025 		res0 |= HCR_NV2;
1026 	if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP))
1027 		res0 |= (HCR_AT | HCR_NV1 | HCR_NV);
1028 	if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
1029 	      kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
1030 		res0 |= (HCR_API | HCR_APK);
1031 	if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP))
1032 		res0 |= BIT(39);
1033 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1034 		res0 |= (HCR_TEA | HCR_TERR);
1035 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1036 		res0 |= HCR_TLOR;
1037 	if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP))
1038 		res1 |= HCR_E2H;
1039 	set_sysreg_masks(kvm, HCR_EL2, res0, res1);
1040 
1041 	/* HCRX_EL2 */
1042 	res0 = HCRX_EL2_RES0;
1043 	res1 = HCRX_EL2_RES1;
1044 	if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP))
1045 		res0 |= HCRX_EL2_PACMEn;
1046 	if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP))
1047 		res0 |= HCRX_EL2_EnFPM;
1048 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1049 		res0 |= HCRX_EL2_GCSEn;
1050 	if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP))
1051 		res0 |= HCRX_EL2_EnIDCP128;
1052 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC))
1053 		res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR);
1054 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP))
1055 		res0 |= HCRX_EL2_TMEA;
1056 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
1057 		res0 |= HCRX_EL2_D128En;
1058 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1059 		res0 |= HCRX_EL2_PTTWI;
1060 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP))
1061 		res0 |= HCRX_EL2_SCTLR2En;
1062 	if (!kvm_has_tcr2(kvm))
1063 		res0 |= HCRX_EL2_TCR2En;
1064 	if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
1065 		res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
1066 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP))
1067 		res0 |= HCRX_EL2_CMOW;
1068 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP))
1069 		res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT);
1070 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) ||
1071 	    !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS))
1072 		res0 |= HCRX_EL2_SMPME;
1073 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))
1074 		res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS);
1075 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V))
1076 		res0 |= HCRX_EL2_EnASR;
1077 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64))
1078 		res0 |= HCRX_EL2_EnALS;
1079 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1080 		res0 |= HCRX_EL2_EnAS0;
1081 	set_sysreg_masks(kvm, HCRX_EL2, res0, res1);
1082 
1083 	/* HFG[RW]TR_EL2 */
1084 	res0 = res1 = 0;
1085 	if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
1086 	      kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
1087 		res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey |
1088 			 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey |
1089 			 HFGxTR_EL2_APIBKey);
1090 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1091 		res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 |
1092 			 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 |
1093 			 HFGxTR_EL2_LORSA_EL1);
1094 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
1095 	    !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
1096 		res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0);
1097 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP))
1098 		res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1;
1099 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1100 		res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 |
1101 			 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 |
1102 			 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 |
1103 			 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 |
1104 			 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1);
1105 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1106 		res0 |= HFGxTR_EL2_nACCDATA_EL1;
1107 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1108 		res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1);
1109 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP))
1110 		res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0);
1111 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1112 		res0 |= HFGxTR_EL2_nRCWMASK_EL1;
1113 	if (!kvm_has_s1pie(kvm))
1114 		res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1);
1115 	if (!kvm_has_s1poe(kvm))
1116 		res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1);
1117 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP))
1118 		res0 |= HFGxTR_EL2_nS2POR_EL1;
1119 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
1120 		res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1);
1121 	set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1);
1122 	set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1);
1123 
1124 	/* HDFG[RW]TR_EL2 */
1125 	res0 = res1 = 0;
1126 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP))
1127 		res0 |= HDFGRTR_EL2_OSDLR_EL1;
1128 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1129 		res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 |
1130 			 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 |
1131 			 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN |
1132 			 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 |
1133 			 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 |
1134 			 HDFGRTR_EL2_PMCEIDn_EL0);
1135 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
1136 		res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 |
1137 			 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 |
1138 			 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 |
1139 			 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 |
1140 			 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 |
1141 			 HDFGRTR_EL2_PMBIDR_EL1);
1142 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1143 		res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS |
1144 			 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM |
1145 			 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID |
1146 			 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR |
1147 			 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR |
1148 			 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR |
1149 			 HDFGRTR_EL2_TRCVICTLR);
1150 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
1151 		res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 |
1152 			 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 |
1153 			 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 |
1154 			 HDFGRTR_EL2_TRBTRG_EL1);
1155 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1156 		res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL |
1157 			 HDFGRTR_EL2_nBRBDATA);
1158 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
1159 		res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1;
1160 	set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1);
1161 
1162 	/* Reuse the bits from the read-side and add the write-specific stuff */
1163 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1164 		res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0);
1165 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1166 		res0 |= HDFGWTR_EL2_TRCOSLAR;
1167 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
1168 		res0 |= HDFGWTR_EL2_TRFCR_EL1;
1169 	set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1);
1170 
1171 	/* HFGITR_EL2 */
1172 	res0 = HFGITR_EL2_RES0;
1173 	res1 = HFGITR_EL2_RES1;
1174 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2))
1175 		res0 |= HFGITR_EL2_DCCVADP;
1176 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2))
1177 		res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP);
1178 	if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
1179 		res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1180 			 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS |
1181 			 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS |
1182 			 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS |
1183 			 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS);
1184 	if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
1185 		res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 |
1186 			 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 |
1187 			 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS |
1188 			 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS |
1189 			 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1190 			 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS);
1191 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP))
1192 		res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX |
1193 			 HFGITR_EL2_CPPRCTX);
1194 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1195 		res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL);
1196 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1197 		res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 |
1198 			 HFGITR_EL2_nGCSEPP);
1199 	if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX))
1200 		res0 |= HFGITR_EL2_COSPRCTX;
1201 	if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP))
1202 		res0 |= HFGITR_EL2_ATS1E1A;
1203 	set_sysreg_masks(kvm, HFGITR_EL2, res0, res1);
1204 
1205 	/* HAFGRTR_EL2 - not a lot to see here */
1206 	res0 = HAFGRTR_EL2_RES0;
1207 	res1 = HAFGRTR_EL2_RES1;
1208 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
1209 		res0 |= ~(res0 | res1);
1210 	set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1);
1211 
1212 	/* TCR2_EL2 */
1213 	res0 = TCR2_EL2_RES0;
1214 	res1 = TCR2_EL2_RES1;
1215 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
1216 		res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128);
1217 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP))
1218 		res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0;
1219 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT))
1220 		res0 |= TCR2_EL2_HAFT;
1221 	if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1222 		res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH;
1223 	if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
1224 		res0 |= TCR2_EL2_AIE;
1225 	if (!kvm_has_s1poe(kvm))
1226 		res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE;
1227 	if (!kvm_has_s1pie(kvm))
1228 		res0 |= TCR2_EL2_PIE;
1229 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1230 		res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 |
1231 			 TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1);
1232 	set_sysreg_masks(kvm, TCR2_EL2, res0, res1);
1233 
1234 	/* SCTLR_EL1 */
1235 	res0 = SCTLR_EL1_RES0;
1236 	res1 = SCTLR_EL1_RES1;
1237 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
1238 		res0 |= SCTLR_EL1_EPAN;
1239 	set_sysreg_masks(kvm, SCTLR_EL1, res0, res1);
1240 
1241 	/* MDCR_EL2 */
1242 	res0 = MDCR_EL2_RES0;
1243 	res1 = MDCR_EL2_RES1;
1244 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1245 		res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR |
1246 			 MDCR_EL2_TPM | MDCR_EL2_HPME);
1247 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
1248 		res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS;
1249 	if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP))
1250 		res0 |= MDCR_EL2_EnSPM;
1251 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1))
1252 		res0 |= MDCR_EL2_HPMD;
1253 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
1254 		res0 |= MDCR_EL2_TTRF;
1255 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5))
1256 		res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP;
1257 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
1258 		res0 |= MDCR_EL2_E2TB;
1259 	if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP))
1260 		res0 |= MDCR_EL2_TDCC;
1261 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) ||
1262 	    kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP))
1263 		res0 |= MDCR_EL2_MTPME;
1264 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7))
1265 		res0 |= MDCR_EL2_HPMFZO;
1266 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP))
1267 		res0 |= MDCR_EL2_PMSSE;
1268 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
1269 		res0 |= MDCR_EL2_HPMFZS;
1270 	if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP))
1271 		res0 |= MDCR_EL2_PMEE;
1272 	if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9))
1273 		res0 |= MDCR_EL2_EBWE;
1274 	if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP))
1275 		res0 |= MDCR_EL2_EnSTEPOP;
1276 	set_sysreg_masks(kvm, MDCR_EL2, res0, res1);
1277 
1278 	/* CNTHCTL_EL2 */
1279 	res0 = GENMASK(63, 20);
1280 	res1 = 0;
1281 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP))
1282 		res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK;
1283 	if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) {
1284 		res0 |= CNTHCTL_ECV;
1285 		if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP))
1286 			res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT |
1287 				 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT);
1288 	}
1289 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1290 		res0 |= GENMASK(11, 8);
1291 	set_sysreg_masks(kvm, CNTHCTL_EL2, res0, res1);
1292 
1293 out:
1294 	for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
1295 		(void)__vcpu_sys_reg(vcpu, sr);
1296 
1297 	return 0;
1298 }
1299 
check_nested_vcpu_requests(struct kvm_vcpu * vcpu)1300 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
1301 {
1302 	if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) {
1303 		struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
1304 
1305 		write_lock(&vcpu->kvm->mmu_lock);
1306 		if (mmu->pending_unmap) {
1307 			kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true);
1308 			mmu->pending_unmap = false;
1309 		}
1310 		write_unlock(&vcpu->kvm->mmu_lock);
1311 	}
1312 }
1313