1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4 * Author: Jintack Lim <jintack.lim@linaro.org>
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10
11 #include <asm/fixmap.h>
12 #include <asm/kvm_arm.h>
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_mmu.h>
15 #include <asm/kvm_nested.h>
16 #include <asm/sysreg.h>
17
18 #include "sys_regs.h"
19
20 struct vncr_tlb {
21 /* The guest's VNCR_EL2 */
22 u64 gva;
23 struct s1_walk_info wi;
24 struct s1_walk_result wr;
25
26 u64 hpa;
27
28 /* -1 when not mapped on a CPU */
29 int cpu;
30
31 /*
32 * true if the TLB is valid. Can only be changed with the
33 * mmu_lock held.
34 */
35 bool valid;
36 };
37
38 /*
39 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
40 * memory usage and potential number of different sets of S2 PTs in
41 * the guests. Running out of S2 MMUs only affects performance (we
42 * will invalidate them more often).
43 */
44 #define S2_MMU_PER_VCPU 2
45
kvm_init_nested(struct kvm * kvm)46 void kvm_init_nested(struct kvm *kvm)
47 {
48 kvm->arch.nested_mmus = NULL;
49 kvm->arch.nested_mmus_size = 0;
50 atomic_set(&kvm->arch.vncr_map_count, 0);
51 }
52
init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)53 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
54 {
55 /*
56 * We only initialise the IPA range on the canonical MMU, which
57 * defines the contract between KVM and userspace on where the
58 * "hardware" is in the IPA space. This affects the validity of MMIO
59 * exits forwarded to userspace, for example.
60 *
61 * For nested S2s, we use the PARange as exposed to the guest, as it
62 * is allowed to use it at will to expose whatever memory map it
63 * wants to its own guests as it would be on real HW.
64 */
65 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
66 }
67
kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)68 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
69 {
70 struct kvm *kvm = vcpu->kvm;
71 struct kvm_s2_mmu *tmp;
72 int num_mmus, ret = 0;
73
74 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&
75 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
76 return -EINVAL;
77
78 if (!vcpu->arch.ctxt.vncr_array)
79 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
80 __GFP_ZERO);
81
82 if (!vcpu->arch.ctxt.vncr_array)
83 return -ENOMEM;
84
85 /*
86 * Let's treat memory allocation failures as benign: If we fail to
87 * allocate anything, return an error and keep the allocated array
88 * alive. Userspace may try to recover by initializing the vcpu
89 * again, and there is no reason to affect the whole VM for this.
90 */
91 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
92
93 if (num_mmus > kvm->arch.nested_mmus_size) {
94 tmp = kvcalloc(num_mmus, sizeof(*tmp), GFP_KERNEL_ACCOUNT);
95 if (!tmp)
96 return -ENOMEM;
97
98 write_lock(&kvm->mmu_lock);
99
100 if (kvm->arch.nested_mmus_size) {
101 memcpy(tmp, kvm->arch.nested_mmus,
102 size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size));
103
104 for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
105 tmp[i].pgt->mmu = &tmp[i];
106 }
107
108 swap(kvm->arch.nested_mmus, tmp);
109
110 write_unlock(&kvm->mmu_lock);
111
112 kvfree(tmp);
113 }
114
115 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
116 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
117
118 if (ret) {
119 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
120 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
121
122 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
123 vcpu->arch.ctxt.vncr_array = NULL;
124
125 return ret;
126 }
127
128 kvm->arch.nested_mmus_size = num_mmus;
129
130 return 0;
131 }
132
133 struct s2_walk_info {
134 u64 baddr;
135 unsigned int max_oa_bits;
136 unsigned int pgshift;
137 unsigned int sl;
138 unsigned int t0sz;
139 bool be;
140 bool ha;
141 };
142
compute_fsc(int level,u32 fsc)143 static u32 compute_fsc(int level, u32 fsc)
144 {
145 return fsc | (level & 0x3);
146 }
147
esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)148 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
149 {
150 u32 esr;
151
152 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
153 esr |= compute_fsc(level, fsc);
154 return esr;
155 }
156
get_ia_size(struct s2_walk_info * wi)157 static int get_ia_size(struct s2_walk_info *wi)
158 {
159 return 64 - wi->t0sz;
160 }
161
check_base_s2_limits(struct kvm_vcpu * vcpu,struct s2_walk_info * wi,int level,int input_size,int stride)162 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
163 int level, int input_size, int stride)
164 {
165 int start_size, pa_max;
166
167 pa_max = kvm_get_pa_bits(vcpu->kvm);
168
169 /* Check translation limits */
170 switch (BIT(wi->pgshift)) {
171 case SZ_64K:
172 if (level == 0 || (level == 1 && pa_max <= 42))
173 return -EFAULT;
174 break;
175 case SZ_16K:
176 if (level == 0 || (level == 1 && pa_max <= 40))
177 return -EFAULT;
178 break;
179 case SZ_4K:
180 if (level < 0 || (level == 0 && pa_max <= 42))
181 return -EFAULT;
182 break;
183 }
184
185 /* Check input size limits */
186 if (input_size > pa_max)
187 return -EFAULT;
188
189 /* Check number of entries in starting level table */
190 start_size = input_size - ((3 - level) * stride + wi->pgshift);
191 if (start_size < 1 || start_size > stride + 4)
192 return -EFAULT;
193
194 return 0;
195 }
196
197 /* Check if output is within boundaries */
check_output_size(struct s2_walk_info * wi,phys_addr_t output)198 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
199 {
200 unsigned int output_size = wi->max_oa_bits;
201
202 if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
203 return -1;
204
205 return 0;
206 }
207
read_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 * desc,struct s2_walk_info * wi)208 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc,
209 struct s2_walk_info *wi)
210 {
211 u64 val;
212 int r;
213
214 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
215 if (r)
216 return r;
217
218 /*
219 * Handle reversedescriptors if endianness differs between the
220 * host and the guest hypervisor.
221 */
222 if (wi->be)
223 *desc = be64_to_cpu((__force __be64)val);
224 else
225 *desc = le64_to_cpu((__force __le64)val);
226
227 return 0;
228 }
229
swap_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 old,u64 new,struct s2_walk_info * wi)230 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,
231 struct s2_walk_info *wi)
232 {
233 if (wi->be) {
234 old = (__force u64)cpu_to_be64(old);
235 new = (__force u64)cpu_to_be64(new);
236 } else {
237 old = (__force u64)cpu_to_le64(old);
238 new = (__force u64)cpu_to_le64(new);
239 }
240
241 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
242 }
243
244 /*
245 * This is essentially a C-version of the pseudo code from the ARM ARM
246 * AArch64.TranslationTableWalk function. I strongly recommend looking at
247 * that pseudocode in trying to understand this.
248 *
249 * Must be called with the kvm->srcu read lock held
250 */
walk_nested_s2_pgd(struct kvm_vcpu * vcpu,phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)251 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
252 struct s2_walk_info *wi, struct kvm_s2_trans *out)
253 {
254 int first_block_level, level, stride, input_size, base_lower_bound;
255 phys_addr_t base_addr;
256 unsigned int addr_top, addr_bottom;
257 u64 desc, new_desc; /* page table entry */
258 int ret;
259 phys_addr_t paddr;
260
261 switch (BIT(wi->pgshift)) {
262 default:
263 case SZ_64K:
264 case SZ_16K:
265 level = 3 - wi->sl;
266 first_block_level = 2;
267 break;
268 case SZ_4K:
269 level = 2 - wi->sl;
270 first_block_level = 1;
271 break;
272 }
273
274 stride = wi->pgshift - 3;
275 input_size = get_ia_size(wi);
276 if (input_size > 48 || input_size < 25)
277 return -EFAULT;
278
279 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);
280 if (WARN_ON(ret)) {
281 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT);
282 return ret;
283 }
284
285 base_lower_bound = 3 + input_size - ((3 - level) * stride +
286 wi->pgshift);
287 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
288
289 if (check_output_size(wi, base_addr)) {
290 /* R_BFHQH */
291 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ);
292 return 1;
293 }
294
295 addr_top = input_size - 1;
296
297 while (1) {
298 phys_addr_t index;
299
300 addr_bottom = (3 - level) * stride + wi->pgshift;
301 index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
302 >> (addr_bottom - 3);
303
304 paddr = base_addr | index;
305 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);
306 if (ret < 0) {
307 out->esr = ESR_ELx_FSC_SEA_TTW(level);
308 return ret;
309 }
310
311 new_desc = desc;
312
313 /* Check for valid descriptor at this point */
314 if (!(desc & KVM_PTE_VALID)) {
315 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
316 out->desc = desc;
317 return 1;
318 }
319
320 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) {
321 if (level < 3)
322 break;
323
324 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
325 out->desc = desc;
326 return 1;
327 }
328
329 /* We're at the final level */
330 if (level == 3)
331 break;
332
333 if (check_output_size(wi, desc)) {
334 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
335 out->desc = desc;
336 return 1;
337 }
338
339 base_addr = desc & GENMASK_ULL(47, wi->pgshift);
340
341 level += 1;
342 addr_top = addr_bottom - 1;
343 }
344
345 if (level < first_block_level) {
346 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
347 out->desc = desc;
348 return 1;
349 }
350
351 if (check_output_size(wi, desc)) {
352 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
353 out->desc = desc;
354 return 1;
355 }
356
357 if (wi->ha)
358 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
359
360 if (new_desc != desc) {
361 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);
362 if (ret)
363 return ret;
364
365 desc = new_desc;
366 }
367
368 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) {
369 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
370 out->desc = desc;
371 return 1;
372 }
373
374 addr_bottom += contiguous_bit_shift(desc, wi, level);
375
376 /* Calculate and return the result */
377 paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
378 (ipa & GENMASK_ULL(addr_bottom - 1, 0));
379 out->output = paddr;
380 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
381 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
382 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
383 out->level = level;
384 out->desc = desc;
385 return 0;
386 }
387
vtcr_to_walk_info(u64 vtcr,struct s2_walk_info * wi)388 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
389 {
390 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
391
392 switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
393 case VTCR_EL2_TG0_4K:
394 wi->pgshift = 12; break;
395 case VTCR_EL2_TG0_16K:
396 wi->pgshift = 14; break;
397 case VTCR_EL2_TG0_64K:
398 default: /* IMPDEF: treat any other value as 64k */
399 wi->pgshift = 16; break;
400 }
401
402 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
403 /* Global limit for now, should eventually be per-VM */
404 wi->max_oa_bits = min(get_kvm_ipa_limit(),
405 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));
406
407 wi->ha = vtcr & VTCR_EL2_HA;
408 }
409
kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)410 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
411 struct kvm_s2_trans *result)
412 {
413 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
414 struct s2_walk_info wi;
415 int ret;
416
417 result->esr = 0;
418
419 if (!vcpu_has_nv(vcpu))
420 return 0;
421
422 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
423
424 vtcr_to_walk_info(vtcr, &wi);
425
426 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
427
428 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
429 if (ret)
430 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
431
432 return ret;
433 }
434
ttl_to_size(u8 ttl)435 static unsigned int ttl_to_size(u8 ttl)
436 {
437 int level = ttl & 3;
438 int gran = (ttl >> 2) & 3;
439 unsigned int max_size = 0;
440
441 switch (gran) {
442 case TLBI_TTL_TG_4K:
443 switch (level) {
444 case 0:
445 break;
446 case 1:
447 max_size = SZ_1G;
448 break;
449 case 2:
450 max_size = SZ_2M;
451 break;
452 case 3:
453 max_size = SZ_4K;
454 break;
455 }
456 break;
457 case TLBI_TTL_TG_16K:
458 switch (level) {
459 case 0:
460 case 1:
461 break;
462 case 2:
463 max_size = SZ_32M;
464 break;
465 case 3:
466 max_size = SZ_16K;
467 break;
468 }
469 break;
470 case TLBI_TTL_TG_64K:
471 switch (level) {
472 case 0:
473 case 1:
474 /* No 52bit IPA support */
475 break;
476 case 2:
477 max_size = SZ_512M;
478 break;
479 case 3:
480 max_size = SZ_64K;
481 break;
482 }
483 break;
484 default: /* No size information */
485 break;
486 }
487
488 return max_size;
489 }
490
pgshift_level_to_ttl(u16 shift,u8 level)491 static u8 pgshift_level_to_ttl(u16 shift, u8 level)
492 {
493 u8 ttl;
494
495 switch(shift) {
496 case 12:
497 ttl = TLBI_TTL_TG_4K;
498 break;
499 case 14:
500 ttl = TLBI_TTL_TG_16K;
501 break;
502 case 16:
503 ttl = TLBI_TTL_TG_64K;
504 break;
505 default:
506 BUG();
507 }
508
509 ttl <<= 2;
510 ttl |= level & 3;
511
512 return ttl;
513 }
514
515 /*
516 * Compute the equivalent of the TTL field by parsing the shadow PT. The
517 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
518 * retrieved from first entry carrying the level as a tag.
519 */
get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)520 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
521 {
522 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
523 kvm_pte_t pte;
524 u8 ttl, level;
525
526 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
527
528 switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
529 case VTCR_EL2_TG0_4K:
530 ttl = (TLBI_TTL_TG_4K << 2);
531 break;
532 case VTCR_EL2_TG0_16K:
533 ttl = (TLBI_TTL_TG_16K << 2);
534 break;
535 case VTCR_EL2_TG0_64K:
536 default: /* IMPDEF: treat any other value as 64k */
537 ttl = (TLBI_TTL_TG_64K << 2);
538 break;
539 }
540
541 tmp = addr;
542
543 again:
544 /* Iteratively compute the block sizes for a particular granule size */
545 switch (FIELD_GET(VTCR_EL2_TG0_MASK, vtcr)) {
546 case VTCR_EL2_TG0_4K:
547 if (sz < SZ_4K) sz = SZ_4K;
548 else if (sz < SZ_2M) sz = SZ_2M;
549 else if (sz < SZ_1G) sz = SZ_1G;
550 else sz = 0;
551 break;
552 case VTCR_EL2_TG0_16K:
553 if (sz < SZ_16K) sz = SZ_16K;
554 else if (sz < SZ_32M) sz = SZ_32M;
555 else sz = 0;
556 break;
557 case VTCR_EL2_TG0_64K:
558 default: /* IMPDEF: treat any other value as 64k */
559 if (sz < SZ_64K) sz = SZ_64K;
560 else if (sz < SZ_512M) sz = SZ_512M;
561 else sz = 0;
562 break;
563 }
564
565 if (sz == 0)
566 return 0;
567
568 tmp &= ~(sz - 1);
569 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
570 goto again;
571 if (!(pte & PTE_VALID))
572 goto again;
573 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
574 if (!level)
575 goto again;
576
577 ttl |= level;
578
579 /*
580 * We now have found some level information in the shadow S2. Check
581 * that the resulting range is actually including the original IPA.
582 */
583 sz = ttl_to_size(ttl);
584 if (addr < (tmp + sz))
585 return ttl;
586
587 return 0;
588 }
589
compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)590 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
591 {
592 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
593 unsigned long max_size;
594 u8 ttl;
595
596 ttl = FIELD_GET(TLBI_TTL_MASK, val);
597
598 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
599 /* No TTL, check the shadow S2 for a hint */
600 u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
601 ttl = get_guest_mapping_ttl(mmu, addr);
602 }
603
604 max_size = ttl_to_size(ttl);
605
606 if (!max_size) {
607 /* Compute the maximum extent of the invalidation */
608 switch (FIELD_GET(VTCR_EL2_TG0_MASK, mmu->tlb_vtcr)) {
609 case VTCR_EL2_TG0_4K:
610 max_size = SZ_1G;
611 break;
612 case VTCR_EL2_TG0_16K:
613 max_size = SZ_32M;
614 break;
615 case VTCR_EL2_TG0_64K:
616 default: /* IMPDEF: treat any other value as 64k */
617 /*
618 * No, we do not support 52bit IPA in nested yet. Once
619 * we do, this should be 4TB.
620 */
621 max_size = SZ_512M;
622 break;
623 }
624 }
625
626 WARN_ON(!max_size);
627 return max_size;
628 }
629
630 /*
631 * We can have multiple *different* MMU contexts with the same VMID:
632 *
633 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
634 *
635 * - Multiple vcpus using private S2s (huh huh...), hence differing by the
636 * VBBTR_EL2.BADDR address
637 *
638 * - A combination of the above...
639 *
640 * We can always identify which MMU context to pick at run-time. However,
641 * TLB invalidation involving a VMID must take action on all the TLBs using
642 * this particular VMID. This translates into applying the same invalidation
643 * operation to all the contexts that are using this VMID. Moar phun!
644 */
kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))645 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
646 const union tlbi_info *info,
647 void (*tlbi_callback)(struct kvm_s2_mmu *,
648 const union tlbi_info *))
649 {
650 write_lock(&kvm->mmu_lock);
651
652 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
653 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
654
655 if (!kvm_s2_mmu_valid(mmu))
656 continue;
657
658 if (vmid == get_vmid(mmu->tlb_vttbr))
659 tlbi_callback(mmu, info);
660 }
661
662 write_unlock(&kvm->mmu_lock);
663 }
664
lookup_s2_mmu(struct kvm_vcpu * vcpu)665 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
666 {
667 struct kvm *kvm = vcpu->kvm;
668 bool nested_stage2_enabled;
669 u64 vttbr, vtcr, hcr;
670
671 lockdep_assert_held_write(&kvm->mmu_lock);
672
673 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
674 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
675 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
676
677 nested_stage2_enabled = hcr & HCR_VM;
678
679 /* Don't consider the CnP bit for the vttbr match */
680 vttbr &= ~VTTBR_CNP_BIT;
681
682 /*
683 * Two possibilities when looking up a S2 MMU context:
684 *
685 * - either S2 is enabled in the guest, and we need a context that is
686 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
687 * which makes it safe from a TLB conflict perspective (a broken
688 * guest won't be able to generate them),
689 *
690 * - or S2 is disabled, and we need a context that is S2-disabled
691 * and matches the VMID only, as all TLBs are tagged by VMID even
692 * if S2 translation is disabled.
693 */
694 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
695 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
696
697 if (!kvm_s2_mmu_valid(mmu))
698 continue;
699
700 if (nested_stage2_enabled &&
701 mmu->nested_stage2_enabled &&
702 vttbr == mmu->tlb_vttbr &&
703 vtcr == mmu->tlb_vtcr)
704 return mmu;
705
706 if (!nested_stage2_enabled &&
707 !mmu->nested_stage2_enabled &&
708 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
709 return mmu;
710 }
711 return NULL;
712 }
713
get_s2_mmu_nested(struct kvm_vcpu * vcpu)714 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
715 {
716 struct kvm *kvm = vcpu->kvm;
717 struct kvm_s2_mmu *s2_mmu;
718 int i;
719
720 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
721
722 s2_mmu = lookup_s2_mmu(vcpu);
723 if (s2_mmu)
724 goto out;
725
726 /*
727 * Make sure we don't always search from the same point, or we
728 * will always reuse a potentially active context, leaving
729 * free contexts unused.
730 */
731 for (i = kvm->arch.nested_mmus_next;
732 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
733 i++) {
734 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
735
736 if (atomic_read(&s2_mmu->refcnt) == 0)
737 break;
738 }
739 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
740
741 /* Set the scene for the next search */
742 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
743
744 /* Make sure we don't forget to do the laundry */
745 if (kvm_s2_mmu_valid(s2_mmu)) {
746 kvm_nested_s2_ptdump_remove_debugfs(s2_mmu);
747 s2_mmu->pending_unmap = true;
748 }
749
750 /*
751 * The virtual VMID (modulo CnP) will be used as a key when matching
752 * an existing kvm_s2_mmu.
753 *
754 * We cache VTCR at allocation time, once and for all. It'd be great
755 * if the guest didn't screw that one up, as this is not very
756 * forgiving...
757 */
758 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
759 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
760 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
761
762 kvm_nested_s2_ptdump_create_debugfs(s2_mmu);
763
764 out:
765 atomic_inc(&s2_mmu->refcnt);
766
767 /*
768 * Set the vCPU request to perform an unmap, even if the pending unmap
769 * originates from another vCPU. This guarantees that the MMU has been
770 * completely unmapped before any vCPU actually uses it, and allows
771 * multiple vCPUs to lend a hand with completing the unmap.
772 */
773 if (s2_mmu->pending_unmap)
774 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu);
775
776 return s2_mmu;
777 }
778
kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)779 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
780 {
781 /* CnP being set denotes an invalid entry */
782 mmu->tlb_vttbr = VTTBR_CNP_BIT;
783 mmu->nested_stage2_enabled = false;
784 atomic_set(&mmu->refcnt, 0);
785 }
786
kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)787 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
788 {
789 /*
790 * If the vCPU kept its reference on the MMU after the last put,
791 * keep rolling with it.
792 */
793 if (is_hyp_ctxt(vcpu)) {
794 if (!vcpu->arch.hw_mmu)
795 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
796 } else {
797 if (!vcpu->arch.hw_mmu) {
798 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
799 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
800 }
801
802 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
803 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
804 }
805 }
806
kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)807 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
808 {
809 /* Unconditionally drop the VNCR mapping if we have one */
810 if (host_data_test_flag(L1_VNCR_MAPPED)) {
811 BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id());
812 BUG_ON(is_hyp_ctxt(vcpu));
813
814 clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu));
815 vcpu->arch.vncr_tlb->cpu = -1;
816 host_data_clear_flag(L1_VNCR_MAPPED);
817 atomic_dec(&vcpu->kvm->arch.vncr_map_count);
818 }
819
820 /*
821 * Keep a reference on the associated stage-2 MMU if the vCPU is
822 * scheduling out and not in WFI emulation, suggesting it is likely to
823 * reuse the MMU sometime soon.
824 */
825 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI))
826 return;
827
828 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
829 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
830
831 vcpu->arch.hw_mmu = NULL;
832 }
833
834 /*
835 * Returns non-zero if permission fault is handled by injecting it to the next
836 * level hypervisor.
837 */
kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)838 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
839 {
840 bool forward_fault = false;
841
842 trans->esr = 0;
843
844 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
845 return 0;
846
847 if (kvm_vcpu_trap_is_iabt(vcpu)) {
848 if (vcpu_mode_priv(vcpu))
849 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
850 else
851 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
852 } else {
853 bool write_fault = kvm_is_write_fault(vcpu);
854
855 forward_fault = ((write_fault && !trans->writable) ||
856 (!write_fault && !trans->readable));
857 }
858
859 if (forward_fault)
860 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
861
862 return forward_fault;
863 }
864
kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)865 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
866 {
867 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
868 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
869
870 return kvm_inject_nested_sync(vcpu, esr_el2);
871 }
872
get_asid_by_regime(struct kvm_vcpu * vcpu,enum trans_regime regime)873 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime)
874 {
875 enum vcpu_sysreg ttbr_elx;
876 u64 tcr;
877 u16 asid;
878
879 switch (regime) {
880 case TR_EL10:
881 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
882 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1;
883 break;
884 case TR_EL20:
885 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
886 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2;
887 break;
888 default:
889 BUG();
890 }
891
892 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx));
893 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
894 !(tcr & TCR_ASID16))
895 asid &= GENMASK(7, 0);
896
897 return asid;
898 }
899
invalidate_vncr(struct vncr_tlb * vt)900 static void invalidate_vncr(struct vncr_tlb *vt)
901 {
902 vt->valid = false;
903 if (vt->cpu != -1)
904 clear_fixmap(vncr_fixmap(vt->cpu));
905 }
906
kvm_invalidate_vncr_ipa(struct kvm * kvm,u64 start,u64 end)907 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
908 {
909 struct kvm_vcpu *vcpu;
910 unsigned long i;
911
912 lockdep_assert_held_write(&kvm->mmu_lock);
913
914 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
915 return;
916
917 kvm_for_each_vcpu(i, vcpu, kvm) {
918 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
919 u64 ipa_start, ipa_end, ipa_size;
920
921 /*
922 * Careful here: We end-up here from an MMU notifier,
923 * and this can race against a vcpu not being onlined
924 * yet, without the pseudo-TLB being allocated.
925 *
926 * Skip those, as they obviously don't participate in
927 * the invalidation at this stage.
928 */
929 if (!vt)
930 continue;
931
932 if (!vt->valid)
933 continue;
934
935 ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
936 vt->wr.level));
937 ipa_start = vt->wr.pa & ~(ipa_size - 1);
938 ipa_end = ipa_start + ipa_size;
939
940 if (ipa_end <= start || ipa_start >= end)
941 continue;
942
943 invalidate_vncr(vt);
944 }
945 }
946
947 struct s1e2_tlbi_scope {
948 enum {
949 TLBI_ALL,
950 TLBI_VA,
951 TLBI_VAA,
952 TLBI_ASID,
953 } type;
954
955 u16 asid;
956 u64 va;
957 u64 size;
958 };
959
invalidate_vncr_va(struct kvm * kvm,struct s1e2_tlbi_scope * scope)960 static void invalidate_vncr_va(struct kvm *kvm,
961 struct s1e2_tlbi_scope *scope)
962 {
963 struct kvm_vcpu *vcpu;
964 unsigned long i;
965
966 lockdep_assert_held_write(&kvm->mmu_lock);
967
968 kvm_for_each_vcpu(i, vcpu, kvm) {
969 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
970 u64 va_start, va_end, va_size;
971
972 if (!vt->valid)
973 continue;
974
975 va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
976 vt->wr.level));
977 va_start = vt->gva & ~(va_size - 1);
978 va_end = va_start + va_size;
979
980 switch (scope->type) {
981 case TLBI_ALL:
982 break;
983
984 case TLBI_VA:
985 if (va_end <= scope->va ||
986 va_start >= (scope->va + scope->size))
987 continue;
988 if (vt->wr.nG && vt->wr.asid != scope->asid)
989 continue;
990 break;
991
992 case TLBI_VAA:
993 if (va_end <= scope->va ||
994 va_start >= (scope->va + scope->size))
995 continue;
996 break;
997
998 case TLBI_ASID:
999 if (!vt->wr.nG || vt->wr.asid != scope->asid)
1000 continue;
1001 break;
1002 }
1003
1004 invalidate_vncr(vt);
1005 }
1006 }
1007
1008 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48)
1009
compute_s1_tlbi_range(struct kvm_vcpu * vcpu,u32 inst,u64 val,struct s1e2_tlbi_scope * scope)1010 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
1011 struct s1e2_tlbi_scope *scope)
1012 {
1013 switch (inst) {
1014 case OP_TLBI_ALLE2:
1015 case OP_TLBI_ALLE2IS:
1016 case OP_TLBI_ALLE2OS:
1017 case OP_TLBI_VMALLE1:
1018 case OP_TLBI_VMALLE1IS:
1019 case OP_TLBI_VMALLE1OS:
1020 case OP_TLBI_ALLE2NXS:
1021 case OP_TLBI_ALLE2ISNXS:
1022 case OP_TLBI_ALLE2OSNXS:
1023 case OP_TLBI_VMALLE1NXS:
1024 case OP_TLBI_VMALLE1ISNXS:
1025 case OP_TLBI_VMALLE1OSNXS:
1026 scope->type = TLBI_ALL;
1027 break;
1028 case OP_TLBI_VAE2:
1029 case OP_TLBI_VAE2IS:
1030 case OP_TLBI_VAE2OS:
1031 case OP_TLBI_VAE1:
1032 case OP_TLBI_VAE1IS:
1033 case OP_TLBI_VAE1OS:
1034 case OP_TLBI_VAE2NXS:
1035 case OP_TLBI_VAE2ISNXS:
1036 case OP_TLBI_VAE2OSNXS:
1037 case OP_TLBI_VAE1NXS:
1038 case OP_TLBI_VAE1ISNXS:
1039 case OP_TLBI_VAE1OSNXS:
1040 case OP_TLBI_VALE2:
1041 case OP_TLBI_VALE2IS:
1042 case OP_TLBI_VALE2OS:
1043 case OP_TLBI_VALE1:
1044 case OP_TLBI_VALE1IS:
1045 case OP_TLBI_VALE1OS:
1046 case OP_TLBI_VALE2NXS:
1047 case OP_TLBI_VALE2ISNXS:
1048 case OP_TLBI_VALE2OSNXS:
1049 case OP_TLBI_VALE1NXS:
1050 case OP_TLBI_VALE1ISNXS:
1051 case OP_TLBI_VALE1OSNXS:
1052 scope->type = TLBI_VA;
1053 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1054 if (!scope->size)
1055 scope->size = SZ_1G;
1056 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1057 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1058 break;
1059 case OP_TLBI_ASIDE1:
1060 case OP_TLBI_ASIDE1IS:
1061 case OP_TLBI_ASIDE1OS:
1062 case OP_TLBI_ASIDE1NXS:
1063 case OP_TLBI_ASIDE1ISNXS:
1064 case OP_TLBI_ASIDE1OSNXS:
1065 scope->type = TLBI_ASID;
1066 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1067 break;
1068 case OP_TLBI_VAAE1:
1069 case OP_TLBI_VAAE1IS:
1070 case OP_TLBI_VAAE1OS:
1071 case OP_TLBI_VAAE1NXS:
1072 case OP_TLBI_VAAE1ISNXS:
1073 case OP_TLBI_VAAE1OSNXS:
1074 case OP_TLBI_VAALE1:
1075 case OP_TLBI_VAALE1IS:
1076 case OP_TLBI_VAALE1OS:
1077 case OP_TLBI_VAALE1NXS:
1078 case OP_TLBI_VAALE1ISNXS:
1079 case OP_TLBI_VAALE1OSNXS:
1080 scope->type = TLBI_VAA;
1081 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1082 if (!scope->size)
1083 scope->size = SZ_1G;
1084 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1085 break;
1086 case OP_TLBI_RVAE2:
1087 case OP_TLBI_RVAE2IS:
1088 case OP_TLBI_RVAE2OS:
1089 case OP_TLBI_RVAE1:
1090 case OP_TLBI_RVAE1IS:
1091 case OP_TLBI_RVAE1OS:
1092 case OP_TLBI_RVAE2NXS:
1093 case OP_TLBI_RVAE2ISNXS:
1094 case OP_TLBI_RVAE2OSNXS:
1095 case OP_TLBI_RVAE1NXS:
1096 case OP_TLBI_RVAE1ISNXS:
1097 case OP_TLBI_RVAE1OSNXS:
1098 case OP_TLBI_RVALE2:
1099 case OP_TLBI_RVALE2IS:
1100 case OP_TLBI_RVALE2OS:
1101 case OP_TLBI_RVALE1:
1102 case OP_TLBI_RVALE1IS:
1103 case OP_TLBI_RVALE1OS:
1104 case OP_TLBI_RVALE2NXS:
1105 case OP_TLBI_RVALE2ISNXS:
1106 case OP_TLBI_RVALE2OSNXS:
1107 case OP_TLBI_RVALE1NXS:
1108 case OP_TLBI_RVALE1ISNXS:
1109 case OP_TLBI_RVALE1OSNXS:
1110 scope->type = TLBI_VA;
1111 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid);
1112 break;
1113 case OP_TLBI_RVAAE1:
1114 case OP_TLBI_RVAAE1IS:
1115 case OP_TLBI_RVAAE1OS:
1116 case OP_TLBI_RVAAE1NXS:
1117 case OP_TLBI_RVAAE1ISNXS:
1118 case OP_TLBI_RVAAE1OSNXS:
1119 case OP_TLBI_RVAALE1:
1120 case OP_TLBI_RVAALE1IS:
1121 case OP_TLBI_RVAALE1OS:
1122 case OP_TLBI_RVAALE1NXS:
1123 case OP_TLBI_RVAALE1ISNXS:
1124 case OP_TLBI_RVAALE1OSNXS:
1125 scope->type = TLBI_VAA;
1126 scope->va = decode_range_tlbi(val, &scope->size, NULL);
1127 break;
1128 }
1129 }
1130
kvm_handle_s1e2_tlbi(struct kvm_vcpu * vcpu,u32 inst,u64 val)1131 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1132 {
1133 struct s1e2_tlbi_scope scope = {};
1134
1135 compute_s1_tlbi_range(vcpu, inst, val, &scope);
1136
1137 guard(write_lock)(&vcpu->kvm->mmu_lock);
1138 invalidate_vncr_va(vcpu->kvm, &scope);
1139 }
1140
kvm_nested_s2_wp(struct kvm * kvm)1141 void kvm_nested_s2_wp(struct kvm *kvm)
1142 {
1143 int i;
1144
1145 lockdep_assert_held_write(&kvm->mmu_lock);
1146
1147 if (!kvm->arch.nested_mmus_size)
1148 return;
1149
1150 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1151 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1152
1153 if (kvm_s2_mmu_valid(mmu))
1154 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
1155 }
1156
1157 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1158 }
1159
kvm_nested_s2_unmap(struct kvm * kvm,bool may_block)1160 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
1161 {
1162 int i;
1163
1164 lockdep_assert_held_write(&kvm->mmu_lock);
1165
1166 if (!kvm->arch.nested_mmus_size)
1167 return;
1168
1169 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1170 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1171
1172 if (kvm_s2_mmu_valid(mmu))
1173 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
1174 }
1175
1176 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1177 }
1178
kvm_nested_s2_flush(struct kvm * kvm)1179 void kvm_nested_s2_flush(struct kvm *kvm)
1180 {
1181 int i;
1182
1183 lockdep_assert_held_write(&kvm->mmu_lock);
1184
1185 if (!kvm->arch.nested_mmus_size)
1186 return;
1187
1188 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1189 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1190
1191 if (kvm_s2_mmu_valid(mmu))
1192 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
1193 }
1194 }
1195
kvm_arch_flush_shadow_all(struct kvm * kvm)1196 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1197 {
1198 int i;
1199
1200 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1201 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1202
1203 if (!WARN_ON(atomic_read(&mmu->refcnt)))
1204 kvm_free_stage2_pgd(mmu);
1205 }
1206 kvfree(kvm->arch.nested_mmus);
1207 kvm->arch.nested_mmus = NULL;
1208 kvm->arch.nested_mmus_size = 0;
1209 kvm_uninit_stage2_mmu(kvm);
1210 }
1211
1212 /*
1213 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter:
1214 *
1215 * - We introduce an internal representation of a vcpu-private TLB,
1216 * representing the mapping between the guest VA contained in VNCR_EL2,
1217 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at.
1218 *
1219 * - On translation fault from a nested VNCR access, we create such a TLB.
1220 * If there is no mapping to describe, the guest inherits the fault.
1221 * Crucially, no actual mapping is done at this stage.
1222 *
1223 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above
1224 * TLB exists, we map it in the fixmap for this CPU, and run with it. We
1225 * have to respect the permissions dictated by the guest, but not the
1226 * memory type (FWB is a must).
1227 *
1228 * - Note that we usually don't do a vcpu_load() on the back of a fault
1229 * (unless we are preempted), so the resolution of a translation fault
1230 * must go via a request that will map the VNCR page in the fixmap.
1231 * vcpu_load() might as well use the same mechanism.
1232 *
1233 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was
1234 * mapped, we unmap it. Yes it is that simple. The TLB still exists
1235 * though, and may be reused at a later load.
1236 *
1237 * - On permission fault, we simply forward the fault to the guest's EL2.
1238 * Get out of my way.
1239 *
1240 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that
1241 * intersects with the TLBI request, invalidate it, and unmap the page
1242 * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1243 * this requires some wide-ranging locking to ensure that nothing races
1244 * against it. This may require some refcounting to avoid the search when
1245 * no such TLB is present.
1246 *
1247 * - On MMU notifiers, we must invalidate our TLB in a similar way, but
1248 * looking at the IPA instead. The funny part is that there may not be a
1249 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST
1250 * instructions.
1251 */
1252
kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu * vcpu)1253 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1254 {
1255 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1256 return 0;
1257
1258 vcpu->arch.vncr_tlb = kzalloc_obj(*vcpu->arch.vncr_tlb,
1259 GFP_KERNEL_ACCOUNT);
1260 if (!vcpu->arch.vncr_tlb)
1261 return -ENOMEM;
1262
1263 return 0;
1264 }
1265
read_vncr_el2(struct kvm_vcpu * vcpu)1266 static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1267 {
1268 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1269 }
1270
kvm_translate_vncr(struct kvm_vcpu * vcpu,bool * is_gmem)1271 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
1272 {
1273 struct kvm_memory_slot *memslot;
1274 bool write_fault, writable;
1275 unsigned long mmu_seq;
1276 struct vncr_tlb *vt;
1277 struct page *page;
1278 u64 va, pfn, gfn;
1279 int ret;
1280
1281 vt = vcpu->arch.vncr_tlb;
1282
1283 /*
1284 * If we're about to walk the EL2 S1 PTs, we must invalidate the
1285 * current TLB, as it could be sampled from another vcpu doing a
1286 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single
1287 * translation, so not much of a choice.
1288 *
1289 * We also prepare the next walk wilst we're at it.
1290 */
1291 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1292 invalidate_vncr(vt);
1293
1294 vt->wi = (struct s1_walk_info) {
1295 .regime = TR_EL20,
1296 .as_el0 = false,
1297 .pan = false,
1298 };
1299 vt->wr = (struct s1_walk_result){};
1300 }
1301
1302 guard(srcu)(&vcpu->kvm->srcu);
1303
1304 va = read_vncr_el2(vcpu);
1305
1306 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1307 if (ret)
1308 return ret;
1309
1310 write_fault = kvm_is_write_fault(vcpu);
1311
1312 mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1313 smp_rmb();
1314
1315 gfn = vt->wr.pa >> PAGE_SHIFT;
1316 memslot = gfn_to_memslot(vcpu->kvm, gfn);
1317 if (!memslot)
1318 return -EFAULT;
1319
1320 *is_gmem = kvm_slot_has_gmem(memslot);
1321 if (!*is_gmem) {
1322 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0,
1323 &writable, &page);
1324 if (is_error_noslot_pfn(pfn) || (write_fault && !writable))
1325 return -EFAULT;
1326 } else {
1327 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
1328 if (ret) {
1329 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
1330 write_fault, false, false);
1331 return ret;
1332 }
1333 }
1334
1335 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1336 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq))
1337 return -EAGAIN;
1338
1339 vt->gva = va;
1340 vt->hpa = pfn << PAGE_SHIFT;
1341 vt->valid = true;
1342 vt->cpu = -1;
1343
1344 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1345 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw);
1346 }
1347
1348 if (vt->wr.pw)
1349 mark_page_dirty(vcpu->kvm, gfn);
1350
1351 return 0;
1352 }
1353
inject_vncr_perm(struct kvm_vcpu * vcpu)1354 static void inject_vncr_perm(struct kvm_vcpu *vcpu)
1355 {
1356 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1357 u64 esr = kvm_vcpu_get_esr(vcpu);
1358
1359 /* Adjust the fault level to reflect that of the guest's */
1360 esr &= ~ESR_ELx_FSC;
1361 esr |= FIELD_PREP(ESR_ELx_FSC,
1362 ESR_ELx_FSC_PERM_L(vt->wr.level));
1363
1364 kvm_inject_nested_sync(vcpu, esr);
1365 }
1366
kvm_vncr_tlb_lookup(struct kvm_vcpu * vcpu)1367 static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu)
1368 {
1369 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1370
1371 lockdep_assert_held_read(&vcpu->kvm->mmu_lock);
1372
1373 if (!vt->valid)
1374 return false;
1375
1376 if (read_vncr_el2(vcpu) != vt->gva)
1377 return false;
1378
1379 if (vt->wr.nG)
1380 return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid;
1381
1382 return true;
1383 }
1384
kvm_handle_vncr_abort(struct kvm_vcpu * vcpu)1385 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1386 {
1387 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1388 u64 esr = kvm_vcpu_get_esr(vcpu);
1389
1390 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR));
1391
1392 if (kvm_vcpu_abt_issea(vcpu))
1393 return kvm_handle_guest_sea(vcpu);
1394
1395 if (esr_fsc_is_permission_fault(esr)) {
1396 inject_vncr_perm(vcpu);
1397 } else if (esr_fsc_is_translation_fault(esr)) {
1398 bool valid, is_gmem = false;
1399 int ret;
1400
1401 scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
1402 valid = kvm_vncr_tlb_lookup(vcpu);
1403
1404 if (!valid)
1405 ret = kvm_translate_vncr(vcpu, &is_gmem);
1406 else
1407 ret = -EPERM;
1408
1409 switch (ret) {
1410 case -EAGAIN:
1411 /* Let's try again... */
1412 break;
1413 case -ENOMEM:
1414 /*
1415 * For guest_memfd, this indicates that it failed to
1416 * create a folio to back the memory. Inform userspace.
1417 */
1418 if (is_gmem)
1419 return 0;
1420 /* Otherwise, let's try again... */
1421 break;
1422 case -EFAULT:
1423 case -EIO:
1424 case -EHWPOISON:
1425 if (is_gmem)
1426 return 0;
1427 fallthrough;
1428 case -EINVAL:
1429 case -ENOENT:
1430 case -EACCES:
1431 /*
1432 * Translation failed, inject the corresponding
1433 * exception back to EL2.
1434 */
1435 BUG_ON(!vt->wr.failed);
1436
1437 esr &= ~ESR_ELx_FSC;
1438 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
1439
1440 kvm_inject_nested_sync(vcpu, esr);
1441 break;
1442 case -EPERM:
1443 /* Hack to deal with POE until we get kernel support */
1444 inject_vncr_perm(vcpu);
1445 break;
1446 case 0:
1447 break;
1448 }
1449 } else {
1450 WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr);
1451 }
1452
1453 return 1;
1454 }
1455
kvm_map_l1_vncr(struct kvm_vcpu * vcpu)1456 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1457 {
1458 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1459 pgprot_t prot;
1460
1461 guard(preempt)();
1462 guard(read_lock)(&vcpu->kvm->mmu_lock);
1463
1464 /*
1465 * The request to map VNCR may have raced against some other
1466 * event, such as an interrupt, and may not be valid anymore.
1467 */
1468 if (is_hyp_ctxt(vcpu))
1469 return;
1470
1471 /*
1472 * Check that the pseudo-TLB is valid and that VNCR_EL2 still
1473 * contains the expected value. If it doesn't, we simply bail out
1474 * without a mapping -- a transformed MSR/MRS will generate the
1475 * fault and allows us to populate the pseudo-TLB.
1476 */
1477 if (!vt->valid)
1478 return;
1479
1480 if (read_vncr_el2(vcpu) != vt->gva)
1481 return;
1482
1483 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid)
1484 return;
1485
1486 vt->cpu = smp_processor_id();
1487
1488 if (vt->wr.pw && vt->wr.pr)
1489 prot = PAGE_KERNEL;
1490 else if (vt->wr.pr)
1491 prot = PAGE_KERNEL_RO;
1492 else
1493 prot = PAGE_NONE;
1494
1495 /*
1496 * We can't map write-only (or no permission at all) in the kernel,
1497 * but the guest can do it if using POE, so we'll have to turn a
1498 * translation fault into a permission fault at runtime.
1499 * FIXME: WO doesn't work at all, need POE support in the kernel.
1500 */
1501 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
1502 __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot);
1503 host_data_set_flag(L1_VNCR_MAPPED);
1504 atomic_inc(&vcpu->kvm->arch.vncr_map_count);
1505 }
1506 }
1507
1508 #define has_tgran_2(__r, __sz) \
1509 ({ \
1510 u64 _s1, _s2, _mmfr0 = __r; \
1511 \
1512 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
1513 TGRAN##__sz##_2, _mmfr0); \
1514 \
1515 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
1516 TGRAN##__sz, _mmfr0); \
1517 \
1518 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \
1519 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \
1520 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \
1521 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \
1522 })
1523 /*
1524 * Our emulated CPU doesn't support all the possible features. For the
1525 * sake of simplicity (and probably mental sanity), wipe out a number
1526 * of feature bits we don't intend to support for the time being.
1527 * This list should get updated as new features get added to the NV
1528 * support, and new extension to the architecture.
1529 */
limit_nv_id_reg(struct kvm * kvm,u32 reg,u64 val)1530 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
1531 {
1532 u64 orig_val = val;
1533
1534 switch (reg) {
1535 case SYS_ID_AA64ISAR1_EL1:
1536 /* Support everything but LS64 and Spec Invalidation */
1537 val &= ~(ID_AA64ISAR1_EL1_LS64 |
1538 ID_AA64ISAR1_EL1_SPECRES);
1539 break;
1540
1541 case SYS_ID_AA64PFR0_EL1:
1542 /* No RME, AMU, MPAM, or S-EL2 */
1543 val &= ~(ID_AA64PFR0_EL1_RME |
1544 ID_AA64PFR0_EL1_AMU |
1545 ID_AA64PFR0_EL1_MPAM |
1546 ID_AA64PFR0_EL1_SEL2 |
1547 ID_AA64PFR0_EL1_EL3 |
1548 ID_AA64PFR0_EL1_EL2 |
1549 ID_AA64PFR0_EL1_EL1 |
1550 ID_AA64PFR0_EL1_EL0);
1551 /* 64bit only at any EL */
1552 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP);
1553 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP);
1554 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP);
1555 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP);
1556 break;
1557
1558 case SYS_ID_AA64PFR1_EL1:
1559 /* Only support BTI, SSBS, CSV2_frac */
1560 val &= ~(ID_AA64PFR1_EL1_PFAR |
1561 ID_AA64PFR1_EL1_MTEX |
1562 ID_AA64PFR1_EL1_THE |
1563 ID_AA64PFR1_EL1_GCS |
1564 ID_AA64PFR1_EL1_MTE_frac |
1565 ID_AA64PFR1_EL1_NMI |
1566 ID_AA64PFR1_EL1_SME |
1567 ID_AA64PFR1_EL1_RES0 |
1568 ID_AA64PFR1_EL1_MPAM_frac |
1569 ID_AA64PFR1_EL1_MTE);
1570 break;
1571
1572 case SYS_ID_AA64PFR2_EL1:
1573 /* GICv5 is not yet supported for NV */
1574 val &= ~ID_AA64PFR2_EL1_GCIE;
1575 break;
1576
1577 case SYS_ID_AA64MMFR0_EL1:
1578 /* Hide ExS, Secure Memory */
1579 val &= ~(ID_AA64MMFR0_EL1_EXS |
1580 ID_AA64MMFR0_EL1_TGRAN4_2 |
1581 ID_AA64MMFR0_EL1_TGRAN16_2 |
1582 ID_AA64MMFR0_EL1_TGRAN64_2 |
1583 ID_AA64MMFR0_EL1_SNSMEM);
1584
1585 /* Hide CNTPOFF if present */
1586 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP);
1587
1588 /* Disallow unsupported S2 page sizes */
1589 switch (PAGE_SIZE) {
1590 case SZ_64K:
1591 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI);
1592 fallthrough;
1593 case SZ_16K:
1594 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI);
1595 fallthrough;
1596 case SZ_4K:
1597 /* Support everything */
1598 break;
1599 }
1600
1601 /*
1602 * Since we can't support a guest S2 page size smaller
1603 * than the host's own page size (due to KVM only
1604 * populating its own S2 using the kernel's page
1605 * size), advertise the limitation using FEAT_GTG.
1606 */
1607 switch (PAGE_SIZE) {
1608 case SZ_4K:
1609 if (has_tgran_2(orig_val, 4))
1610 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP);
1611 fallthrough;
1612 case SZ_16K:
1613 if (has_tgran_2(orig_val, 16))
1614 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP);
1615 fallthrough;
1616 case SZ_64K:
1617 if (has_tgran_2(orig_val, 64))
1618 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP);
1619 break;
1620 }
1621
1622 /* Cap PARange to 48bits */
1623 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48);
1624 break;
1625
1626 case SYS_ID_AA64MMFR1_EL1:
1627 val &= ~(ID_AA64MMFR1_EL1_CMOW |
1628 ID_AA64MMFR1_EL1_nTLBPA |
1629 ID_AA64MMFR1_EL1_ETS);
1630
1631 /* FEAT_E2H0 implies no VHE */
1632 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))
1633 val &= ~ID_AA64MMFR1_EL1_VH;
1634
1635 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF);
1636 break;
1637
1638 case SYS_ID_AA64MMFR2_EL1:
1639 val &= ~(ID_AA64MMFR2_EL1_BBM |
1640 ID_AA64MMFR2_EL1_TTL |
1641 GENMASK_ULL(47, 44) |
1642 ID_AA64MMFR2_EL1_ST |
1643 ID_AA64MMFR2_EL1_CCIDX |
1644 ID_AA64MMFR2_EL1_VARange);
1645
1646 /* Force TTL support */
1647 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP);
1648 break;
1649
1650 case SYS_ID_AA64MMFR4_EL1:
1651 /*
1652 * You get EITHER
1653 *
1654 * - FEAT_VHE without FEAT_E2H0
1655 * - FEAT_NV limited to FEAT_NV2
1656 * - HCR_EL2.NV1 being RES0
1657 *
1658 * OR
1659 *
1660 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV
1661 *
1662 * Life is too short for anything else.
1663 */
1664 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) {
1665 val = 0;
1666 } else {
1667 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY);
1668 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1);
1669 }
1670 break;
1671
1672 case SYS_ID_AA64DFR0_EL1:
1673 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */
1674 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff |
1675 ID_AA64DFR0_EL1_BRBE |
1676 ID_AA64DFR0_EL1_MTPMU |
1677 ID_AA64DFR0_EL1_TraceBuffer |
1678 ID_AA64DFR0_EL1_TraceFilt |
1679 ID_AA64DFR0_EL1_PMSVer |
1680 ID_AA64DFR0_EL1_CTX_CMPs |
1681 ID_AA64DFR0_EL1_SEBEP |
1682 ID_AA64DFR0_EL1_PMSS |
1683 ID_AA64DFR0_EL1_TraceVer);
1684
1685 /*
1686 * FEAT_Debugv8p9 requires support for extended breakpoints /
1687 * watchpoints.
1688 */
1689 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1690 break;
1691 }
1692
1693 return val;
1694 }
1695
kvm_vcpu_apply_reg_masks(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr,u64 v)1696 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu,
1697 enum vcpu_sysreg sr, u64 v)
1698 {
1699 struct resx resx;
1700
1701 resx = kvm_get_sysreg_resx(vcpu->kvm, sr);
1702 v &= ~resx.res0;
1703 v |= resx.res1;
1704
1705 return v;
1706 }
1707
set_sysreg_masks(struct kvm * kvm,int sr,struct resx resx)1708 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx)
1709 {
1710 BUILD_BUG_ON(!__builtin_constant_p(sr));
1711 BUILD_BUG_ON(sr < __SANITISED_REG_START__);
1712 BUILD_BUG_ON(sr >= NR_SYS_REGS);
1713
1714 kvm_set_sysreg_resx(kvm, sr, resx);
1715 }
1716
kvm_init_nv_sysregs(struct kvm_vcpu * vcpu)1717 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
1718 {
1719 struct kvm *kvm = vcpu->kvm;
1720 struct resx resx;
1721
1722 lockdep_assert_held(&kvm->arch.config_lock);
1723
1724 if (kvm->arch.sysreg_masks)
1725 goto out;
1726
1727 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks),
1728 GFP_KERNEL_ACCOUNT);
1729 if (!kvm->arch.sysreg_masks)
1730 return -ENOMEM;
1731
1732 /* VTTBR_EL2 */
1733 resx = (typeof(resx)){};
1734 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
1735 resx.res0 |= GENMASK(63, 56);
1736 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
1737 resx.res0 |= VTTBR_CNP_BIT;
1738 set_sysreg_masks(kvm, VTTBR_EL2, resx);
1739
1740 /* VTCR_EL2 */
1741 resx = get_reg_fixed_bits(kvm, VTCR_EL2);
1742 set_sysreg_masks(kvm, VTCR_EL2, resx);
1743
1744 /* VMPIDR_EL2 */
1745 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24);
1746 resx.res1 = BIT(31);
1747 set_sysreg_masks(kvm, VMPIDR_EL2, resx);
1748
1749 /* HCR_EL2 */
1750 resx = get_reg_fixed_bits(kvm, HCR_EL2);
1751 set_sysreg_masks(kvm, HCR_EL2, resx);
1752
1753 /* HCRX_EL2 */
1754 resx = get_reg_fixed_bits(kvm, HCRX_EL2);
1755 set_sysreg_masks(kvm, HCRX_EL2, resx);
1756
1757 /* HFG[RW]TR_EL2 */
1758 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2);
1759 set_sysreg_masks(kvm, HFGRTR_EL2, resx);
1760 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2);
1761 set_sysreg_masks(kvm, HFGWTR_EL2, resx);
1762
1763 /* HDFG[RW]TR_EL2 */
1764 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2);
1765 set_sysreg_masks(kvm, HDFGRTR_EL2, resx);
1766 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2);
1767 set_sysreg_masks(kvm, HDFGWTR_EL2, resx);
1768
1769 /* HFGITR_EL2 */
1770 resx = get_reg_fixed_bits(kvm, HFGITR_EL2);
1771 set_sysreg_masks(kvm, HFGITR_EL2, resx);
1772
1773 /* HAFGRTR_EL2 - not a lot to see here */
1774 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2);
1775 set_sysreg_masks(kvm, HAFGRTR_EL2, resx);
1776
1777 /* HFG[RW]TR2_EL2 */
1778 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2);
1779 set_sysreg_masks(kvm, HFGRTR2_EL2, resx);
1780 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2);
1781 set_sysreg_masks(kvm, HFGWTR2_EL2, resx);
1782
1783 /* HDFG[RW]TR2_EL2 */
1784 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2);
1785 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx);
1786 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2);
1787 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx);
1788
1789 /* HFGITR2_EL2 */
1790 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2);
1791 set_sysreg_masks(kvm, HFGITR2_EL2, resx);
1792
1793 /* TCR2_EL2 */
1794 resx = get_reg_fixed_bits(kvm, TCR2_EL2);
1795 set_sysreg_masks(kvm, TCR2_EL2, resx);
1796
1797 /* SCTLR_EL1 */
1798 resx = get_reg_fixed_bits(kvm, SCTLR_EL1);
1799 set_sysreg_masks(kvm, SCTLR_EL1, resx);
1800
1801 /* SCTLR_EL2 */
1802 resx = get_reg_fixed_bits(kvm, SCTLR_EL2);
1803 set_sysreg_masks(kvm, SCTLR_EL2, resx);
1804
1805 /* SCTLR2_ELx */
1806 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1);
1807 set_sysreg_masks(kvm, SCTLR2_EL1, resx);
1808 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2);
1809 set_sysreg_masks(kvm, SCTLR2_EL2, resx);
1810
1811 /* MDCR_EL2 */
1812 resx = get_reg_fixed_bits(kvm, MDCR_EL2);
1813 set_sysreg_masks(kvm, MDCR_EL2, resx);
1814
1815 /* CNTHCTL_EL2 */
1816 resx.res0 = GENMASK(63, 20);
1817 resx.res1 = 0;
1818 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP))
1819 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK;
1820 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) {
1821 resx.res0 |= CNTHCTL_ECV;
1822 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP))
1823 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT |
1824 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT);
1825 }
1826 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1827 resx.res0 |= GENMASK(11, 8);
1828 set_sysreg_masks(kvm, CNTHCTL_EL2, resx);
1829
1830 /* ICH_HCR_EL2 */
1831 resx.res0 = ICH_HCR_EL2_RES0;
1832 resx.res1 = ICH_HCR_EL2_RES1;
1833 if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS))
1834 resx.res0 |= ICH_HCR_EL2_TDIR;
1835 /* No GICv4 is presented to the guest */
1836 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;
1837 set_sysreg_masks(kvm, ICH_HCR_EL2, resx);
1838
1839 /* VNCR_EL2 */
1840 resx.res0 = VNCR_EL2_RES0;
1841 resx.res1 = VNCR_EL2_RES1;
1842 set_sysreg_masks(kvm, VNCR_EL2, resx);
1843
1844 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */
1845 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4);
1846 resx.res1 = ZCR_ELx_RES1;
1847 set_sysreg_masks(kvm, ZCR_EL2, resx);
1848
1849 out:
1850 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
1851 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0);
1852
1853 return 0;
1854 }
1855
check_nested_vcpu_requests(struct kvm_vcpu * vcpu)1856 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
1857 {
1858 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) {
1859 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
1860
1861 write_lock(&vcpu->kvm->mmu_lock);
1862 if (mmu->pending_unmap) {
1863 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true);
1864 mmu->pending_unmap = false;
1865 }
1866 write_unlock(&vcpu->kvm->mmu_lock);
1867 }
1868
1869 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
1870 kvm_map_l1_vncr(vcpu);
1871
1872 /* Must be last, as may switch context! */
1873 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
1874 kvm_inject_nested_irq(vcpu);
1875 }
1876
1877 /*
1878 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
1879 * can write to HCR_EL2 behind our back, potentially changing the exception
1880 * routing / masking for even the host context.
1881 *
1882 * What follows is some slop to (1) react to exception routing / masking and (2)
1883 * preserve the pending SError state across translation regimes.
1884 */
kvm_nested_flush_hwstate(struct kvm_vcpu * vcpu)1885 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
1886 {
1887 if (!vcpu_has_nv(vcpu))
1888 return;
1889
1890 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1891 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1892 }
1893
kvm_nested_sync_hwstate(struct kvm_vcpu * vcpu)1894 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
1895 {
1896 unsigned long *hcr = vcpu_hcr(vcpu);
1897
1898 if (!vcpu_has_nv(vcpu))
1899 return;
1900
1901 /*
1902 * We previously decided that an SError was deliverable to the guest.
1903 * Reap the pending state from HCR_EL2 and...
1904 */
1905 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
1906 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
1907
1908 /*
1909 * Re-attempt SError injection in case the deliverability has changed,
1910 * which is necessary to faithfully emulate WFI the case of a pending
1911 * SError being a wakeup condition.
1912 */
1913 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1914 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1915 }
1916
1917 /*
1918 * KVM unconditionally sets most of these traps anyway but use an allowlist
1919 * to document the guest hypervisor traps that may take precedence and guard
1920 * against future changes to the non-nested trap configuration.
1921 */
1922 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \
1923 MDCR_EL2_TDA | \
1924 MDCR_EL2_TDRA | \
1925 MDCR_EL2_TTRF | \
1926 MDCR_EL2_TPMS | \
1927 MDCR_EL2_TPM | \
1928 MDCR_EL2_TPMCR | \
1929 MDCR_EL2_TDCC | \
1930 MDCR_EL2_TDOSA)
1931
kvm_nested_setup_mdcr_el2(struct kvm_vcpu * vcpu)1932 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu)
1933 {
1934 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
1935
1936 if (is_nested_ctxt(vcpu))
1937 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
1938 /*
1939 * In yet another example where FEAT_NV2 is fscking broken, accesses
1940 * to MDSCR_EL1 are redirected to the VNCR despite having an effect
1941 * at EL2. Use a big hammer to apply sanity.
1942 *
1943 * Unless of course we have FEAT_FGT, in which case we can precisely
1944 * trap MDSCR_EL1.
1945 */
1946 else if (!cpus_have_final_cap(ARM64_HAS_FGT))
1947 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
1948 }
1949