1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4 * Author: Jintack Lim <jintack.lim@linaro.org>
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10
11 #include <asm/fixmap.h>
12 #include <asm/kvm_arm.h>
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_mmu.h>
15 #include <asm/kvm_nested.h>
16 #include <asm/sysreg.h>
17
18 #include "sys_regs.h"
19 #include "vgic/vgic.h"
20
21 struct vncr_tlb {
22 /* The guest's VNCR_EL2 */
23 u64 gva;
24 struct s1_walk_info wi;
25 struct s1_walk_result wr;
26
27 u64 hpa;
28 bool hpa_writable;
29
30 /* -1 when not mapped on a CPU */
31 atomic_t cpu;
32
33 /*
34 * true if the TLB is valid. Can only be changed with the
35 * mmu_lock held.
36 */
37 bool valid;
38 };
39
40 /*
41 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
42 * memory usage and potential number of different sets of S2 PTs in
43 * the guests. Running out of S2 MMUs only affects performance (we
44 * will invalidate them more often).
45 */
46 #define S2_MMU_PER_VCPU 2
47
kvm_init_nested(struct kvm * kvm)48 void kvm_init_nested(struct kvm *kvm)
49 {
50 kvm->arch.nested_mmus = NULL;
51 kvm->arch.nested_mmus_size = 0;
52 atomic_set(&kvm->arch.vncr_tlb_count, 0);
53 }
54
init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)55 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
56 {
57 /*
58 * We only initialise the IPA range on the canonical MMU, which
59 * defines the contract between KVM and userspace on where the
60 * "hardware" is in the IPA space. This affects the validity of MMIO
61 * exits forwarded to userspace, for example.
62 *
63 * For nested S2s, we use the PARange as exposed to the guest, as it
64 * is allowed to use it at will to expose whatever memory map it
65 * wants to its own guests as it would be on real HW.
66 */
67 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
68 }
69
kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)70 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
71 {
72 struct kvm *kvm = vcpu->kvm;
73 struct kvm_s2_mmu *tmp;
74 int num_mmus, ret = 0;
75
76 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&
77 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
78 return -EINVAL;
79
80 if (!vcpu->arch.ctxt.vncr_array)
81 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
82 __GFP_ZERO);
83
84 if (!vcpu->arch.ctxt.vncr_array)
85 return -ENOMEM;
86
87 /*
88 * Let's treat memory allocation failures as benign: If we fail to
89 * allocate anything, return an error and keep the allocated array
90 * alive. Userspace may try to recover by initializing the vcpu
91 * again, and there is no reason to affect the whole VM for this.
92 */
93 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
94
95 if (num_mmus > kvm->arch.nested_mmus_size) {
96 tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT);
97 if (!tmp)
98 return -ENOMEM;
99
100 write_lock(&kvm->mmu_lock);
101
102 if (kvm->arch.nested_mmus_size) {
103 memcpy(tmp, kvm->arch.nested_mmus,
104 size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size));
105
106 for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
107 tmp[i].pgt->mmu = &tmp[i];
108 }
109
110 swap(kvm->arch.nested_mmus, tmp);
111
112 write_unlock(&kvm->mmu_lock);
113
114 kvfree(tmp);
115 }
116
117 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
118 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
119
120 if (ret) {
121 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
122 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
123
124 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
125 vcpu->arch.ctxt.vncr_array = NULL;
126
127 return ret;
128 }
129
130 kvm->arch.nested_mmus_size = num_mmus;
131
132 return 0;
133 }
134
135 struct s2_walk_info {
136 u64 baddr;
137 unsigned int max_oa_bits;
138 unsigned int pgshift;
139 unsigned int sl;
140 unsigned int t0sz;
141 bool be;
142 bool ha;
143 };
144
compute_fsc(int level,u32 fsc)145 static u32 compute_fsc(int level, u32 fsc)
146 {
147 return fsc | (level & 0x3);
148 }
149
esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)150 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
151 {
152 u32 esr;
153
154 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
155 esr |= compute_fsc(level, fsc);
156 return esr;
157 }
158
get_ia_size(struct s2_walk_info * wi)159 static int get_ia_size(struct s2_walk_info *wi)
160 {
161 return 64 - wi->t0sz;
162 }
163
check_base_s2_limits(struct kvm_vcpu * vcpu,struct s2_walk_info * wi,int level,int input_size,int stride)164 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
165 int level, int input_size, int stride)
166 {
167 int start_size, pa_max;
168
169 pa_max = kvm_get_pa_bits(vcpu->kvm);
170
171 /* Check translation limits */
172 switch (BIT(wi->pgshift)) {
173 case SZ_64K:
174 if (level == 0 || (level == 1 && pa_max <= 42))
175 return -EFAULT;
176 break;
177 case SZ_16K:
178 if (level == 0 || (level == 1 && pa_max <= 40))
179 return -EFAULT;
180 break;
181 case SZ_4K:
182 if (level < 0 || (level == 0 && pa_max <= 42))
183 return -EFAULT;
184 break;
185 }
186
187 /* Check input size limits */
188 if (input_size > pa_max)
189 return -EFAULT;
190
191 /* Check number of entries in starting level table */
192 start_size = input_size - ((3 - level) * stride + wi->pgshift);
193 if (start_size < 1 || start_size > stride + 4)
194 return -EFAULT;
195
196 return 0;
197 }
198
199 /* Check if output is within boundaries */
check_output_size(struct s2_walk_info * wi,phys_addr_t output)200 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
201 {
202 unsigned int output_size = wi->max_oa_bits;
203
204 if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
205 return -1;
206
207 return 0;
208 }
209
read_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 * desc,struct s2_walk_info * wi)210 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc,
211 struct s2_walk_info *wi)
212 {
213 u64 val;
214 int r;
215
216 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
217 if (r)
218 return r;
219
220 /*
221 * Handle reversedescriptors if endianness differs between the
222 * host and the guest hypervisor.
223 */
224 if (wi->be)
225 *desc = be64_to_cpu((__force __be64)val);
226 else
227 *desc = le64_to_cpu((__force __le64)val);
228
229 return 0;
230 }
231
swap_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 old,u64 new,struct s2_walk_info * wi)232 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,
233 struct s2_walk_info *wi)
234 {
235 if (wi->be) {
236 old = (__force u64)cpu_to_be64(old);
237 new = (__force u64)cpu_to_be64(new);
238 } else {
239 old = (__force u64)cpu_to_le64(old);
240 new = (__force u64)cpu_to_le64(new);
241 }
242
243 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
244 }
245
246 /*
247 * This is essentially a C-version of the pseudo code from the ARM ARM
248 * AArch64.TranslationTableWalk function. I strongly recommend looking at
249 * that pseudocode in trying to understand this.
250 *
251 * Must be called with the kvm->srcu read lock held
252 */
walk_nested_s2_pgd(struct kvm_vcpu * vcpu,phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)253 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
254 struct s2_walk_info *wi, struct kvm_s2_trans *out)
255 {
256 int first_block_level, level, stride, input_size, base_lower_bound;
257 phys_addr_t base_addr;
258 unsigned int addr_top, addr_bottom;
259 u64 desc, new_desc; /* page table entry */
260 int ret;
261 phys_addr_t paddr;
262
263 switch (BIT(wi->pgshift)) {
264 default:
265 case SZ_64K:
266 case SZ_16K:
267 level = 3 - wi->sl;
268 first_block_level = 2;
269 break;
270 case SZ_4K:
271 level = 2 - wi->sl;
272 first_block_level = 1;
273 break;
274 }
275
276 stride = wi->pgshift - 3;
277 input_size = get_ia_size(wi);
278 if (input_size > 48 || input_size < 25)
279 return -EFAULT;
280
281 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);
282 if (WARN_ON(ret)) {
283 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT);
284 return ret;
285 }
286
287 base_lower_bound = 3 + input_size - ((3 - level) * stride +
288 wi->pgshift);
289 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
290
291 if (check_output_size(wi, base_addr)) {
292 /* R_BFHQH */
293 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ);
294 return 1;
295 }
296
297 addr_top = input_size - 1;
298
299 while (1) {
300 phys_addr_t index;
301
302 addr_bottom = (3 - level) * stride + wi->pgshift;
303 index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
304 >> (addr_bottom - 3);
305
306 paddr = base_addr | index;
307 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);
308 if (ret < 0) {
309 out->esr = ESR_ELx_FSC_SEA_TTW(level);
310 return ret;
311 }
312
313 new_desc = desc;
314
315 /* Check for valid descriptor at this point */
316 if (!(desc & KVM_PTE_VALID)) {
317 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
318 out->desc = desc;
319 return 1;
320 }
321
322 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) {
323 if (level < 3)
324 break;
325
326 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
327 out->desc = desc;
328 return 1;
329 }
330
331 /* We're at the final level */
332 if (level == 3)
333 break;
334
335 if (check_output_size(wi, desc)) {
336 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
337 out->desc = desc;
338 return 1;
339 }
340
341 base_addr = desc & GENMASK_ULL(47, wi->pgshift);
342
343 level += 1;
344 addr_top = addr_bottom - 1;
345 }
346
347 if (level < first_block_level) {
348 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
349 out->desc = desc;
350 return 1;
351 }
352
353 if (check_output_size(wi, desc)) {
354 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
355 out->desc = desc;
356 return 1;
357 }
358
359 if (wi->ha)
360 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
361
362 if (new_desc != desc) {
363 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);
364 if (ret == -EAGAIN)
365 return ret;
366 if (ret) {
367 out->esr = ESR_ELx_FSC_SEA_TTW(level);
368 out->desc = desc;
369 return 1;
370 }
371
372 desc = new_desc;
373 }
374
375 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) {
376 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
377 out->desc = desc;
378 return 1;
379 }
380
381 addr_bottom += contiguous_bit_shift(desc, wi, level);
382
383 /* Calculate and return the result */
384 paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
385 (ipa & GENMASK_ULL(addr_bottom - 1, 0));
386 out->output = paddr;
387 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
388 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
389 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
390 out->level = level;
391 out->desc = desc;
392 return 0;
393 }
394
395 #define _has_tgran_2(__r, __sz) \
396 ({ \
397 u64 _s1, _s2, _mmfr0 = __r; \
398 \
399 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
400 TGRAN##__sz##_2, _mmfr0); \
401 \
402 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
403 TGRAN##__sz, _mmfr0); \
404 \
405 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \
406 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \
407 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \
408 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \
409 })
410
has_tgran_2(u64 mmfr0,unsigned int shift)411 static bool has_tgran_2(u64 mmfr0, unsigned int shift)
412 {
413 switch (shift) {
414 case 12:
415 return _has_tgran_2(mmfr0, 4);
416 case 14:
417 return _has_tgran_2(mmfr0, 16);
418 case 16:
419 return _has_tgran_2(mmfr0, 64);
420 default:
421 BUG();
422 }
423 }
424
fallback_tgran2_shift(u64 mmfr0)425 static unsigned int fallback_tgran2_shift(u64 mmfr0)
426 {
427 if (has_tgran_2(mmfr0, PAGE_SHIFT))
428 return PAGE_SHIFT;
429 else if (has_tgran_2(mmfr0, 12))
430 return 12;
431 else if (has_tgran_2(mmfr0, 14))
432 return 14;
433 else if (has_tgran_2(mmfr0, 16))
434 return 16;
435 else
436 return PAGE_SHIFT;
437 }
438
vtcr_to_tg0_pgshift(struct kvm * kvm,u64 vtcr)439 static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr)
440 {
441 u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr);
442 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
443 unsigned int shift;
444
445 switch (tg0) {
446 case VTCR_EL2_TG0_4K:
447 shift = 12;
448 break;
449 case VTCR_EL2_TG0_16K:
450 shift = 14;
451 break;
452 case VTCR_EL2_TG0_64K:
453 /* IMPDEF: treat any other value as 64k, subject to fallback */
454 default:
455 shift = 16;
456 }
457
458 /*
459 * If TGx is programmed to an unimplemented value (not advertised in
460 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is
461 * written, as per the architecture. Choose an available one while
462 * prioritizing PAGE_SIZE.
463 */
464 if (!has_tgran_2(mmfr0, shift))
465 return fallback_tgran2_shift(mmfr0);
466
467 return shift;
468 }
469
vtcr_to_tg0_pgsize(struct kvm * kvm,u64 vtcr)470 static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr)
471 {
472 return BIT(vtcr_to_tg0_pgshift(kvm, vtcr));
473 }
474
setup_s2_walk(struct kvm_vcpu * vcpu,struct s2_walk_info * wi)475 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi)
476 {
477 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
478
479 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
480 wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK;
481 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);
482 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
483 /* Global limit for now, should eventually be per-VM */
484 wi->max_oa_bits = min(get_kvm_ipa_limit(),
485 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));
486 wi->ha = vtcr & VTCR_EL2_HA;
487 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
488 }
489
kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)490 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
491 struct kvm_s2_trans *result)
492 {
493 struct s2_walk_info wi;
494 int ret;
495
496 result->esr = 0;
497
498 if (!vcpu_has_nv(vcpu))
499 return 0;
500
501 setup_s2_walk(vcpu, &wi);
502
503 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
504 if (ret)
505 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
506
507 return ret;
508 }
509
__ttl_to_size(u8 ttl)510 static unsigned int __ttl_to_size(u8 ttl)
511 {
512 int level = ttl & 3;
513 int gran = (ttl >> 2) & 3;
514 unsigned int max_size = 0;
515
516 switch (gran) {
517 case TLBI_TTL_TG_4K:
518 switch (level) {
519 case 0:
520 break;
521 case 1:
522 max_size = SZ_1G;
523 break;
524 case 2:
525 max_size = SZ_2M;
526 break;
527 case 3:
528 max_size = SZ_4K;
529 break;
530 }
531 break;
532 case TLBI_TTL_TG_16K:
533 switch (level) {
534 case 0:
535 case 1:
536 break;
537 case 2:
538 max_size = SZ_32M;
539 break;
540 case 3:
541 max_size = SZ_16K;
542 break;
543 }
544 break;
545 case TLBI_TTL_TG_64K:
546 switch (level) {
547 case 0:
548 case 1:
549 /* No 52bit IPA support */
550 break;
551 case 2:
552 max_size = SZ_512M;
553 break;
554 case 3:
555 max_size = SZ_64K;
556 break;
557 }
558 break;
559 default: /* No size information */
560 break;
561 }
562
563 return max_size;
564 }
565
ttl_to_size(u8 ttl)566 static unsigned int ttl_to_size(u8 ttl)
567 {
568 return __ttl_to_size(ttl) ?: SZ_1G;
569 }
570
pgshift_level_to_ttl(u16 shift,s8 level)571 static u8 pgshift_level_to_ttl(u16 shift, s8 level)
572 {
573 u8 ttl;
574
575 /*
576 * If we don't have a proper level, fallback to the maximum
577 * size.
578 */
579 if (level < 0)
580 return 0;
581
582 switch(shift) {
583 case 12:
584 ttl = TLBI_TTL_TG_4K;
585 break;
586 case 14:
587 ttl = TLBI_TTL_TG_16K;
588 break;
589 case 16:
590 ttl = TLBI_TTL_TG_64K;
591 break;
592 default:
593 BUG();
594 }
595
596 ttl <<= 2;
597 ttl |= level & 3;
598
599 return ttl;
600 }
601
602 /*
603 * Compute the equivalent of the TTL field by parsing the shadow PT. The
604 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
605 * retrieved from first entry carrying the level as a tag.
606 */
get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)607 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
608 {
609 size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr);
610 u64 tmp, sz = 0;
611 kvm_pte_t pte;
612 u8 ttl, level;
613
614 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
615
616 switch (tg0_size) {
617 case SZ_4K:
618 ttl = (TLBI_TTL_TG_4K << 2);
619 break;
620 case SZ_16K:
621 ttl = (TLBI_TTL_TG_16K << 2);
622 break;
623 case SZ_64K:
624 default: /* IMPDEF: treat any other value as 64k */
625 ttl = (TLBI_TTL_TG_64K << 2);
626 break;
627 }
628
629 tmp = addr;
630
631 again:
632 /* Iteratively compute the block sizes for a particular granule size */
633 switch (tg0_size) {
634 case SZ_4K:
635 if (sz < SZ_4K) sz = SZ_4K;
636 else if (sz < SZ_2M) sz = SZ_2M;
637 else if (sz < SZ_1G) sz = SZ_1G;
638 else sz = 0;
639 break;
640 case SZ_16K:
641 if (sz < SZ_16K) sz = SZ_16K;
642 else if (sz < SZ_32M) sz = SZ_32M;
643 else sz = 0;
644 break;
645 case SZ_64K:
646 default: /* IMPDEF: treat any other value as 64k */
647 if (sz < SZ_64K) sz = SZ_64K;
648 else if (sz < SZ_512M) sz = SZ_512M;
649 else sz = 0;
650 break;
651 }
652
653 if (sz == 0)
654 return 0;
655
656 tmp &= ~(sz - 1);
657 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
658 goto again;
659 if (!(pte & PTE_VALID))
660 goto again;
661 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
662 if (!level)
663 goto again;
664
665 ttl |= level;
666
667 /*
668 * We now have found some level information in the shadow S2. Check
669 * that the resulting range is actually including the original IPA.
670 */
671 sz = ttl_to_size(ttl);
672 if (addr < (tmp + sz))
673 return ttl;
674
675 return 0;
676 }
677
compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)678 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
679 {
680 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
681 unsigned long max_size;
682 u8 ttl;
683
684 ttl = FIELD_GET(TLBI_TTL_MASK, val);
685
686 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
687 /* No TTL, check the shadow S2 for a hint */
688 u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
689 ttl = get_guest_mapping_ttl(mmu, addr);
690 }
691
692 /*
693 * Don't use the default 1GB fallback, as we can adapt to the
694 * max mapping size we allow at S2.
695 */
696 max_size = __ttl_to_size(ttl);
697
698 if (!max_size) {
699 /* Compute the maximum extent of the invalidation */
700 switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) {
701 case SZ_4K:
702 max_size = SZ_1G;
703 break;
704 case SZ_16K:
705 max_size = SZ_32M;
706 break;
707 case SZ_64K:
708 default: /* IMPDEF: treat any other value as 64k */
709 /*
710 * No, we do not support 52bit IPA in nested yet. Once
711 * we do, this should be 4TB.
712 */
713 max_size = SZ_512M;
714 break;
715 }
716 }
717
718 WARN_ON(!max_size);
719 return max_size;
720 }
721
722 /*
723 * We can have multiple *different* MMU contexts with the same VMID:
724 *
725 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
726 *
727 * - Multiple vcpus using private S2s (huh huh...), hence differing by the
728 * VBBTR_EL2.BADDR address
729 *
730 * - A combination of the above...
731 *
732 * We can always identify which MMU context to pick at run-time. However,
733 * TLB invalidation involving a VMID must take action on all the TLBs using
734 * this particular VMID. This translates into applying the same invalidation
735 * operation to all the contexts that are using this VMID. Moar phun!
736 */
kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))737 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
738 const union tlbi_info *info,
739 void (*tlbi_callback)(struct kvm_s2_mmu *,
740 const union tlbi_info *))
741 {
742 write_lock(&kvm->mmu_lock);
743
744 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
745 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
746
747 if (!kvm_s2_mmu_valid(mmu))
748 continue;
749
750 if (vmid == get_vmid(mmu->tlb_vttbr))
751 tlbi_callback(mmu, info);
752 }
753
754 write_unlock(&kvm->mmu_lock);
755 }
756
lookup_s2_mmu(struct kvm_vcpu * vcpu)757 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
758 {
759 struct kvm *kvm = vcpu->kvm;
760 bool nested_stage2_enabled;
761 u64 vttbr, vtcr, hcr;
762
763 lockdep_assert_held_write(&kvm->mmu_lock);
764
765 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
766 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
767 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
768
769 nested_stage2_enabled = hcr & HCR_VM;
770
771 /* Don't consider the CnP bit for the vttbr match */
772 vttbr &= ~VTTBR_CNP_BIT;
773
774 /*
775 * Two possibilities when looking up a S2 MMU context:
776 *
777 * - either S2 is enabled in the guest, and we need a context that is
778 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
779 * which makes it safe from a TLB conflict perspective (a broken
780 * guest won't be able to generate them),
781 *
782 * - or S2 is disabled, and we need a context that is S2-disabled
783 * and matches the VMID only, as all TLBs are tagged by VMID even
784 * if S2 translation is disabled.
785 */
786 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
787 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
788
789 if (!kvm_s2_mmu_valid(mmu))
790 continue;
791
792 if (nested_stage2_enabled &&
793 mmu->nested_stage2_enabled &&
794 vttbr == mmu->tlb_vttbr &&
795 vtcr == mmu->tlb_vtcr)
796 return mmu;
797
798 if (!nested_stage2_enabled &&
799 !mmu->nested_stage2_enabled &&
800 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
801 return mmu;
802 }
803 return NULL;
804 }
805
get_s2_mmu_nested(struct kvm_vcpu * vcpu)806 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
807 {
808 struct kvm *kvm = vcpu->kvm;
809 struct kvm_s2_mmu *s2_mmu;
810 int i;
811
812 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
813
814 s2_mmu = lookup_s2_mmu(vcpu);
815 if (s2_mmu)
816 goto out;
817
818 /*
819 * Make sure we don't always search from the same point, or we
820 * will always reuse a potentially active context, leaving
821 * free contexts unused.
822 */
823 for (i = kvm->arch.nested_mmus_next;
824 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
825 i++) {
826 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
827
828 if (atomic_read(&s2_mmu->refcnt) == 0)
829 break;
830 }
831 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
832
833 /* Set the scene for the next search */
834 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
835
836 /* Make sure we don't forget to do the laundry */
837 if (kvm_s2_mmu_valid(s2_mmu)) {
838 kvm_nested_s2_ptdump_remove_debugfs(s2_mmu);
839 s2_mmu->pending_unmap = true;
840 }
841
842 /*
843 * The virtual VMID (modulo CnP) will be used as a key when matching
844 * an existing kvm_s2_mmu.
845 *
846 * We cache VTCR at allocation time, once and for all. It'd be great
847 * if the guest didn't screw that one up, as this is not very
848 * forgiving...
849 */
850 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
851 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
852 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
853
854 kvm_nested_s2_ptdump_create_debugfs(s2_mmu);
855
856 out:
857 atomic_inc(&s2_mmu->refcnt);
858
859 /*
860 * Set the vCPU request to perform an unmap, even if the pending unmap
861 * originates from another vCPU. This guarantees that the MMU has been
862 * completely unmapped before any vCPU actually uses it, and allows
863 * multiple vCPUs to lend a hand with completing the unmap.
864 */
865 if (s2_mmu->pending_unmap)
866 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu);
867
868 return s2_mmu;
869 }
870
kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)871 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
872 {
873 /* CnP being set denotes an invalid entry */
874 mmu->tlb_vttbr = VTTBR_CNP_BIT;
875 mmu->nested_stage2_enabled = false;
876 atomic_set(&mmu->refcnt, 0);
877 }
878
kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)879 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
880 {
881 /*
882 * If the vCPU kept its reference on the MMU after the last put,
883 * keep rolling with it.
884 */
885 if (is_hyp_ctxt(vcpu)) {
886 if (!vcpu->arch.hw_mmu)
887 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
888 } else {
889 if (!vcpu->arch.hw_mmu) {
890 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
891 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
892 }
893
894 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
895 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
896 }
897 }
898
899 /*
900 * Unmapping an L1 VNCR can happen concurrently without the mmu lock being
901 * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures
902 * that only one CPU sets it to -1 while getting a valid CPU number back.
903 */
unmap_l1_vncr(struct vncr_tlb * vt)904 static int unmap_l1_vncr(struct vncr_tlb *vt)
905 {
906 int cpu = atomic_xchg_relaxed(&vt->cpu, -1);
907
908 if (cpu != -1)
909 clear_fixmap(vncr_fixmap(cpu));
910
911 return cpu;
912 }
913
this_cpu_reset_vncr_fixmap(struct kvm_vcpu * vcpu)914 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu)
915 {
916 if (!host_data_test_flag(L1_VNCR_MAPPED))
917 return;
918
919 BUG_ON(is_hyp_ctxt(vcpu));
920
921 /*
922 * Unconditionally unmap the local VNCR if we have lost the race
923 * against a concurrent TLBI. Otherwise we could end-up running
924 * another vcpu with VNCR still mapped if the TLBI thread is
925 * preempted between the exchange and the clear_fixmap().
926 *
927 * Note that we do not care about the TLBI nuking the fixmap behind
928 * the back of an running vcpu. This will only generate a fault and
929 * possibly a retranslation.
930 */
931 if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1)
932 clear_fixmap(vncr_fixmap(smp_processor_id()));
933 host_data_clear_flag(L1_VNCR_MAPPED);
934 }
935
kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)936 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
937 {
938 /* Unconditionally drop the VNCR mapping if we have one */
939 this_cpu_reset_vncr_fixmap(vcpu);
940
941 /*
942 * Keep a reference on the associated stage-2 MMU if the vCPU is
943 * scheduling out and not in WFI emulation, suggesting it is likely to
944 * reuse the MMU sometime soon.
945 */
946 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI))
947 return;
948
949 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
950 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
951
952 vcpu->arch.hw_mmu = NULL;
953 }
954
955 /*
956 * Returns non-zero if permission fault is handled by injecting it to the next
957 * level hypervisor.
958 */
kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)959 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
960 {
961 bool forward_fault = false;
962
963 trans->esr = 0;
964
965 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
966 return 0;
967
968 if (kvm_vcpu_trap_is_iabt(vcpu)) {
969 if (vcpu_mode_priv(vcpu))
970 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
971 else
972 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
973 } else {
974 bool write_fault = kvm_is_write_fault(vcpu);
975
976 forward_fault = ((write_fault && !trans->writable) ||
977 (!write_fault && !trans->readable));
978 }
979
980 if (forward_fault)
981 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
982
983 return forward_fault;
984 }
985
kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)986 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
987 {
988 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
989 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
990
991 return kvm_inject_nested_sync(vcpu, esr_el2);
992 }
993
get_asid_by_regime(struct kvm_vcpu * vcpu,enum trans_regime regime)994 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime)
995 {
996 enum vcpu_sysreg ttbr_elx;
997 u64 tcr;
998 u16 asid;
999
1000 switch (regime) {
1001 case TR_EL10:
1002 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
1003 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1;
1004 break;
1005 case TR_EL20:
1006 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1007 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2;
1008 break;
1009 default:
1010 BUG();
1011 }
1012
1013 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx));
1014 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1015 !(tcr & TCR_ASID16))
1016 asid &= GENMASK(7, 0);
1017
1018 return asid;
1019 }
1020
invalidate_vncr(struct kvm * kvm,struct vncr_tlb * vt)1021 static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt)
1022 {
1023 BUG_ON(!vt->valid);
1024 vt->valid = false;
1025 unmap_l1_vncr(vt);
1026 atomic_dec(&kvm->arch.vncr_tlb_count);
1027 }
1028
vncr_tlb_intersects(struct vncr_tlb * vt,u64 addr,u64 scope_start,u64 scope_size)1029 static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr,
1030 u64 scope_start, u64 scope_size)
1031 {
1032 u64 tlb_size, tlb_start, tlb_end, scope_end;
1033
1034 tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level));
1035
1036 tlb_start = addr & ~(tlb_size - 1);
1037 tlb_end = tlb_start + tlb_size - 1;
1038 scope_end = scope_start + scope_size - 1;
1039
1040 return !(tlb_end < scope_start || tlb_start > scope_end);
1041 }
1042
1043 /*
1044 * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and
1045 * either can race against a vcpu not being onlined yet (no pseudo-TLB
1046 * allocated). Similarly, the TLB might be invalid. Skip those, as they
1047 * obviously don't participate in the invalidation at this stage.
1048 */
1049 #define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \
1050 kvm_for_each_vcpu(idx, vcpup, kvm) \
1051 if (((tlbp) = vcpup->arch.vncr_tlb) && \
1052 (tlbp)->valid)
1053
kvm_invalidate_vncr_ipa(struct kvm * kvm,u64 start,u64 end)1054 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
1055 {
1056 struct kvm_vcpu *vcpu;
1057 struct vncr_tlb *vt;
1058 unsigned long i;
1059
1060 lockdep_assert_held_write(&kvm->mmu_lock);
1061
1062 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1063 return;
1064
1065 /*
1066 * Note that invalidating the VNCR on the back of an MMU notifier
1067 * doesn't require messing with the invalidation counter for a
1068 * parallel walk. The notifier itself will have bumped the counter,
1069 * making sure we rewalk.
1070 */
1071 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm)
1072 if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start))
1073 invalidate_vncr(kvm, vt);
1074 }
1075
1076 struct s1e2_tlbi_scope {
1077 enum {
1078 TLBI_ALL,
1079 TLBI_VA,
1080 TLBI_VAA,
1081 TLBI_ASID,
1082 } type;
1083
1084 u16 asid;
1085 u64 va;
1086 u64 size;
1087 };
1088
invalidate_vncr_va(struct kvm * kvm,struct s1e2_tlbi_scope * scope)1089 static void invalidate_vncr_va(struct kvm *kvm,
1090 struct s1e2_tlbi_scope *scope)
1091 {
1092 struct kvm_vcpu *vcpu;
1093 struct vncr_tlb *vt;
1094 unsigned long i;
1095
1096 lockdep_assert_held_write(&kvm->mmu_lock);
1097
1098 /*
1099 * We might be performing a parallel S1 walk, so bump up the
1100 * invalidation counter even in the absence of an actual VNCR TLB
1101 * invalidation, as this could indicate that the guest has gone
1102 * through a BBM sequence.
1103 */
1104 kvm->mmu_invalidate_seq++;
1105 smp_wmb();
1106
1107 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
1108 switch (scope->type) {
1109 case TLBI_ALL:
1110 break;
1111
1112 case TLBI_VA:
1113 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size))
1114 continue;
1115 if (vt->wr.nG && vt->wr.asid != scope->asid)
1116 continue;
1117 break;
1118
1119 case TLBI_VAA:
1120 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size))
1121 continue;
1122 break;
1123
1124 case TLBI_ASID:
1125 if (!vt->wr.nG || vt->wr.asid != scope->asid)
1126 continue;
1127 break;
1128 }
1129
1130 invalidate_vncr(kvm, vt);
1131 }
1132 }
1133
1134 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48)
1135
compute_s1_tlbi_range(struct kvm_vcpu * vcpu,u32 inst,u64 val,struct s1e2_tlbi_scope * scope)1136 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
1137 struct s1e2_tlbi_scope *scope)
1138 {
1139 switch (inst) {
1140 case OP_TLBI_ALLE2:
1141 case OP_TLBI_ALLE2IS:
1142 case OP_TLBI_ALLE2OS:
1143 case OP_TLBI_VMALLE1:
1144 case OP_TLBI_VMALLE1IS:
1145 case OP_TLBI_VMALLE1OS:
1146 case OP_TLBI_ALLE2NXS:
1147 case OP_TLBI_ALLE2ISNXS:
1148 case OP_TLBI_ALLE2OSNXS:
1149 case OP_TLBI_VMALLE1NXS:
1150 case OP_TLBI_VMALLE1ISNXS:
1151 case OP_TLBI_VMALLE1OSNXS:
1152 scope->type = TLBI_ALL;
1153 break;
1154 case OP_TLBI_VAE2:
1155 case OP_TLBI_VAE2IS:
1156 case OP_TLBI_VAE2OS:
1157 case OP_TLBI_VAE1:
1158 case OP_TLBI_VAE1IS:
1159 case OP_TLBI_VAE1OS:
1160 case OP_TLBI_VAE2NXS:
1161 case OP_TLBI_VAE2ISNXS:
1162 case OP_TLBI_VAE2OSNXS:
1163 case OP_TLBI_VAE1NXS:
1164 case OP_TLBI_VAE1ISNXS:
1165 case OP_TLBI_VAE1OSNXS:
1166 case OP_TLBI_VALE2:
1167 case OP_TLBI_VALE2IS:
1168 case OP_TLBI_VALE2OS:
1169 case OP_TLBI_VALE1:
1170 case OP_TLBI_VALE1IS:
1171 case OP_TLBI_VALE1OS:
1172 case OP_TLBI_VALE2NXS:
1173 case OP_TLBI_VALE2ISNXS:
1174 case OP_TLBI_VALE2OSNXS:
1175 case OP_TLBI_VALE1NXS:
1176 case OP_TLBI_VALE1ISNXS:
1177 case OP_TLBI_VALE1OSNXS:
1178 scope->type = TLBI_VA;
1179 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1180 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1181 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1182 break;
1183 case OP_TLBI_ASIDE1:
1184 case OP_TLBI_ASIDE1IS:
1185 case OP_TLBI_ASIDE1OS:
1186 case OP_TLBI_ASIDE1NXS:
1187 case OP_TLBI_ASIDE1ISNXS:
1188 case OP_TLBI_ASIDE1OSNXS:
1189 scope->type = TLBI_ASID;
1190 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1191 break;
1192 case OP_TLBI_VAAE1:
1193 case OP_TLBI_VAAE1IS:
1194 case OP_TLBI_VAAE1OS:
1195 case OP_TLBI_VAAE1NXS:
1196 case OP_TLBI_VAAE1ISNXS:
1197 case OP_TLBI_VAAE1OSNXS:
1198 case OP_TLBI_VAALE1:
1199 case OP_TLBI_VAALE1IS:
1200 case OP_TLBI_VAALE1OS:
1201 case OP_TLBI_VAALE1NXS:
1202 case OP_TLBI_VAALE1ISNXS:
1203 case OP_TLBI_VAALE1OSNXS:
1204 scope->type = TLBI_VAA;
1205 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1206 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1207 break;
1208 case OP_TLBI_RVAE2:
1209 case OP_TLBI_RVAE2IS:
1210 case OP_TLBI_RVAE2OS:
1211 case OP_TLBI_RVAE1:
1212 case OP_TLBI_RVAE1IS:
1213 case OP_TLBI_RVAE1OS:
1214 case OP_TLBI_RVAE2NXS:
1215 case OP_TLBI_RVAE2ISNXS:
1216 case OP_TLBI_RVAE2OSNXS:
1217 case OP_TLBI_RVAE1NXS:
1218 case OP_TLBI_RVAE1ISNXS:
1219 case OP_TLBI_RVAE1OSNXS:
1220 case OP_TLBI_RVALE2:
1221 case OP_TLBI_RVALE2IS:
1222 case OP_TLBI_RVALE2OS:
1223 case OP_TLBI_RVALE1:
1224 case OP_TLBI_RVALE1IS:
1225 case OP_TLBI_RVALE1OS:
1226 case OP_TLBI_RVALE2NXS:
1227 case OP_TLBI_RVALE2ISNXS:
1228 case OP_TLBI_RVALE2OSNXS:
1229 case OP_TLBI_RVALE1NXS:
1230 case OP_TLBI_RVALE1ISNXS:
1231 case OP_TLBI_RVALE1OSNXS:
1232 scope->type = TLBI_VA;
1233 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid);
1234 break;
1235 case OP_TLBI_RVAAE1:
1236 case OP_TLBI_RVAAE1IS:
1237 case OP_TLBI_RVAAE1OS:
1238 case OP_TLBI_RVAAE1NXS:
1239 case OP_TLBI_RVAAE1ISNXS:
1240 case OP_TLBI_RVAAE1OSNXS:
1241 case OP_TLBI_RVAALE1:
1242 case OP_TLBI_RVAALE1IS:
1243 case OP_TLBI_RVAALE1OS:
1244 case OP_TLBI_RVAALE1NXS:
1245 case OP_TLBI_RVAALE1ISNXS:
1246 case OP_TLBI_RVAALE1OSNXS:
1247 scope->type = TLBI_VAA;
1248 scope->va = decode_range_tlbi(val, &scope->size, NULL);
1249 break;
1250 }
1251 }
1252
kvm_handle_s1e2_tlbi(struct kvm_vcpu * vcpu,u32 inst,u64 val)1253 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1254 {
1255 struct s1e2_tlbi_scope scope = {};
1256
1257 compute_s1_tlbi_range(vcpu, inst, val, &scope);
1258
1259 guard(write_lock)(&vcpu->kvm->mmu_lock);
1260 invalidate_vncr_va(vcpu->kvm, &scope);
1261 }
1262
kvm_nested_s2_wp(struct kvm * kvm)1263 void kvm_nested_s2_wp(struct kvm *kvm)
1264 {
1265 int i;
1266
1267 lockdep_assert_held_write(&kvm->mmu_lock);
1268
1269 if (!kvm->arch.nested_mmus_size)
1270 return;
1271
1272 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1273 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1274
1275 if (kvm_s2_mmu_valid(mmu))
1276 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
1277 }
1278
1279 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1280 }
1281
kvm_nested_s2_unmap(struct kvm * kvm,bool may_block)1282 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
1283 {
1284 int i;
1285
1286 lockdep_assert_held_write(&kvm->mmu_lock);
1287
1288 if (!kvm->arch.nested_mmus_size)
1289 return;
1290
1291 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1292 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1293
1294 if (kvm_s2_mmu_valid(mmu))
1295 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
1296 }
1297
1298 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1299 }
1300
kvm_nested_s2_flush(struct kvm * kvm)1301 void kvm_nested_s2_flush(struct kvm *kvm)
1302 {
1303 int i;
1304
1305 lockdep_assert_held_write(&kvm->mmu_lock);
1306
1307 if (!kvm->arch.nested_mmus_size)
1308 return;
1309
1310 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1311 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1312
1313 if (kvm_s2_mmu_valid(mmu))
1314 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
1315 }
1316 }
1317
kvm_arch_flush_shadow_all(struct kvm * kvm)1318 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1319 {
1320 int i;
1321
1322 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1323 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1324
1325 if (!WARN_ON(atomic_read(&mmu->refcnt)))
1326 kvm_free_stage2_pgd(mmu);
1327 }
1328 kvfree(kvm->arch.nested_mmus);
1329 kvm->arch.nested_mmus = NULL;
1330 kvm->arch.nested_mmus_size = 0;
1331 kvm_uninit_stage2_mmu(kvm);
1332 }
1333
1334 /*
1335 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter:
1336 *
1337 * - We introduce an internal representation of a vcpu-private TLB,
1338 * representing the mapping between the guest VA contained in VNCR_EL2,
1339 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at.
1340 *
1341 * - On translation fault from a nested VNCR access, we create such a TLB.
1342 * If there is no mapping to describe, the guest inherits the fault.
1343 * Crucially, no actual mapping is done at this stage.
1344 *
1345 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above
1346 * TLB exists, we map it in the fixmap for this CPU, and run with it. We
1347 * have to respect the permissions dictated by the guest, but not the
1348 * memory type (FWB is a must).
1349 *
1350 * - Note that we usually don't do a vcpu_load() on the back of a fault
1351 * (unless we are preempted), so the resolution of a translation fault
1352 * must go via a request that will map the VNCR page in the fixmap.
1353 * vcpu_load() might as well use the same mechanism.
1354 *
1355 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was
1356 * mapped, we unmap it. Yes it is that simple. The TLB still exists
1357 * though, and may be reused at a later load.
1358 *
1359 * - On permission fault, we simply forward the fault to the guest's EL2.
1360 * Get out of my way.
1361 *
1362 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that
1363 * intersects with the TLBI request, invalidate it, and unmap the page
1364 * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1365 * this requires some wide-ranging locking to ensure that nothing races
1366 * against it. This requires some refcounting to avoid the search when
1367 * no such TLB is present (see below).
1368 *
1369 * - On MMU notifiers, we must invalidate our TLB in a similar way, but
1370 * looking at the IPA instead. The funny part is that there may not be a
1371 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST
1372 * instructions.
1373 *
1374 * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't
1375 * the number of *mapped* L1 VNCR pages, which is likely be a subset (and
1376 * by definition, a TLBI handled from L1 runs with the canonical VNCR
1377 * page, not the L1's). The innermost trap handling code checks this to
1378 * find out whether to return to the guest ASAP (no L1 TLBs) or to visit
1379 * this part of the world for some extra invalidation work.
1380 */
1381
kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu * vcpu)1382 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1383 {
1384 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1385 return 0;
1386
1387 if (!vcpu->arch.vncr_tlb) {
1388 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb,
1389 GFP_KERNEL_ACCOUNT);
1390
1391 /*
1392 * Taking the lock on assignment ensures that the TLB is
1393 * seen as initialised when following the pointer (release
1394 * semantics of the unlock), and avoids having acquires on
1395 * each user which already take the lock.
1396 */
1397 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
1398 vcpu->arch.vncr_tlb = vt;
1399 }
1400
1401 if (!vcpu->arch.vncr_tlb)
1402 return -ENOMEM;
1403
1404 return 0;
1405 }
1406
read_vncr_el2(struct kvm_vcpu * vcpu)1407 static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1408 {
1409 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1410 }
1411
kvm_translate_vncr(struct kvm_vcpu * vcpu,bool * is_gmem)1412 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
1413 {
1414 struct kvm_memory_slot *memslot;
1415 bool write_fault, writable;
1416 unsigned long mmu_seq;
1417 struct vncr_tlb *vt;
1418 struct page *page;
1419 u64 va, pfn, gfn;
1420 int ret;
1421
1422 vt = vcpu->arch.vncr_tlb;
1423
1424 /*
1425 * If we're about to walk the EL2 S1 PTs, we must invalidate the
1426 * current TLB, as it could be sampled from another vcpu doing a
1427 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single
1428 * translation, so not much of a choice.
1429 *
1430 * We also prepare the next walk wilst we're at it.
1431 */
1432 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1433 this_cpu_reset_vncr_fixmap(vcpu);
1434 if (vt->valid)
1435 invalidate_vncr(vcpu->kvm, vt);
1436
1437 vt->wi = (struct s1_walk_info) {
1438 .regime = TR_EL20,
1439 .as_el0 = false,
1440 .pan = false,
1441 };
1442 vt->wr = (struct s1_walk_result){};
1443 }
1444
1445 guard(srcu)(&vcpu->kvm->srcu);
1446
1447 va = read_vncr_el2(vcpu);
1448
1449 mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1450 smp_rmb();
1451
1452 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1453 if (ret)
1454 return ret;
1455
1456 write_fault = kvm_is_write_fault(vcpu);
1457
1458 gfn = vt->wr.pa >> PAGE_SHIFT;
1459 memslot = gfn_to_memslot(vcpu->kvm, gfn);
1460 if (!memslot) {
1461 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1462 return -EFAULT;
1463 }
1464
1465 *is_gmem = kvm_slot_has_gmem(memslot);
1466 if (!*is_gmem) {
1467 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0,
1468 &writable, &page);
1469 if (is_error_noslot_pfn(pfn)) {
1470 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1471 return -EFAULT;
1472 }
1473 } else {
1474 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
1475 if (ret) {
1476 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
1477 write_fault, false, false);
1478 return ret;
1479 }
1480
1481 writable = !(memslot->flags & KVM_MEM_READONLY);
1482 }
1483
1484 /*
1485 * FIXME: This check is too restrictive as KVM allows cacheable memory
1486 * attributes for PFNMAP VMAs that have cacheable attributes in host
1487 * stage-1.
1488 */
1489 if (!pfn_is_map_memory(pfn)) {
1490 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1491 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1492 return -EINVAL;
1493 }
1494
1495 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1496 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
1497 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1498 return -EAGAIN;
1499 }
1500
1501 vt->gva = va;
1502 vt->hpa = pfn << PAGE_SHIFT;
1503 vt->hpa_writable = writable;
1504 vt->valid = true;
1505 atomic_set(&vt->cpu, -1);
1506
1507 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1508 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
1509 }
1510
1511 if (vt->wr.pw && vt->hpa_writable)
1512 mark_page_dirty(vcpu->kvm, gfn);
1513
1514 return 0;
1515 }
1516
handle_vncr_perm(struct kvm_vcpu * vcpu)1517 static void handle_vncr_perm(struct kvm_vcpu *vcpu)
1518 {
1519 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1520 u64 esr = kvm_vcpu_get_esr(vcpu);
1521 u64 fsc;
1522
1523 /*
1524 * Promote to an external abort if the stage-1 permits writes but the
1525 * HPA is read-only (e.g. RO memslot).
1526 */
1527 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable)
1528 fsc = ESR_ELx_FSC_EXTABT;
1529 /*
1530 * Otherwise, inject a permission fault using the guest's translation
1531 * level rather than the host's.
1532 */
1533 else
1534 fsc = ESR_ELx_FSC_PERM_L(vt->wr.level);
1535
1536 esr &= ~ESR_ELx_FSC;
1537 esr |= FIELD_PREP(ESR_ELx_FSC, fsc);
1538
1539 kvm_inject_nested_sync(vcpu, esr);
1540 }
1541
kvm_handle_vncr_abort(struct kvm_vcpu * vcpu)1542 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1543 {
1544 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1545 u64 esr = kvm_vcpu_get_esr(vcpu);
1546 bool is_gmem = false;
1547 bool perm;
1548 int ret;
1549
1550 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR));
1551
1552 if (kvm_vcpu_abt_issea(vcpu))
1553 return kvm_handle_guest_sea(vcpu);
1554
1555 if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) {
1556 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr);
1557 return -EIO;
1558 }
1559
1560 /*
1561 * Speculatively increment the TLB count to make sure concurrent
1562 * TLBIs will take the slow path, and will interact with the retry
1563 * mechanism. Drop it again on error.
1564 */
1565 atomic_inc(&vcpu->kvm->arch.vncr_tlb_count);
1566 smp_mb__after_atomic();
1567
1568 ret = kvm_translate_vncr(vcpu, &is_gmem);
1569 if (ret) {
1570 smp_mb__before_atomic();
1571 atomic_dec(&vcpu->kvm->arch.vncr_tlb_count);
1572 }
1573
1574 switch (ret) {
1575 case -EAGAIN:
1576 /* Let's try again... */
1577 return 1;
1578 case -ENOMEM:
1579 /*
1580 * For guest_memfd, this indicates that it failed to
1581 * create a folio to back the memory. Inform userspace.
1582 */
1583 if (is_gmem)
1584 return 0;
1585 /* Otherwise, let's try again... */
1586 break;
1587 case -EFAULT:
1588 case -EIO:
1589 case -EHWPOISON:
1590 if (is_gmem)
1591 return 0;
1592 fallthrough;
1593 case -EINVAL:
1594 case -ENOENT:
1595 case -EACCES:
1596 /*
1597 * Translation failed, inject the corresponding
1598 * exception back to EL2.
1599 */
1600 esr &= ~ESR_ELx_FSC;
1601 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
1602
1603 kvm_inject_nested_sync(vcpu, esr);
1604 break;
1605 case 0:
1606 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr;
1607 if (!perm)
1608 handle_vncr_perm(vcpu);
1609 break;
1610 }
1611
1612 return 1;
1613 }
1614
kvm_map_l1_vncr(struct kvm_vcpu * vcpu)1615 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1616 {
1617 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1618 pgprot_t prot;
1619
1620 guard(preempt)();
1621 guard(read_lock)(&vcpu->kvm->mmu_lock);
1622
1623 /*
1624 * The request to map VNCR may have raced against some other
1625 * event, such as an interrupt, and may not be valid anymore.
1626 */
1627 if (is_hyp_ctxt(vcpu))
1628 return;
1629
1630 /*
1631 * Check that the pseudo-TLB is valid and that VNCR_EL2 still
1632 * contains the expected value. If it doesn't, we simply bail out
1633 * without a mapping -- a transformed MSR/MRS will generate the
1634 * fault and allows us to populate the pseudo-TLB.
1635 */
1636 if (!vt->valid)
1637 return;
1638
1639 /* We cache the MMU state in the TLB. Check that it matches. */
1640 if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr))
1641 return;
1642
1643 if (read_vncr_el2(vcpu) != vt->gva)
1644 return;
1645
1646 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid)
1647 return;
1648
1649 if (vt->hpa_writable && vt->wr.pw && vt->wr.pr)
1650 prot = PAGE_KERNEL;
1651 else if (vt->wr.pr)
1652 prot = PAGE_KERNEL_RO;
1653 else
1654 prot = PAGE_NONE;
1655
1656 /*
1657 * We can't map write-only (or no permission at all) in the kernel,
1658 * but the guest can do it if using POE, so we'll have to turn a
1659 * translation fault into a permission fault at runtime.
1660 * FIXME: WO doesn't work at all, need POE support in the kernel.
1661 */
1662 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
1663 atomic_set(&vt->cpu, smp_processor_id());
1664 __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot);
1665 host_data_set_flag(L1_VNCR_MAPPED);
1666 }
1667 }
1668
1669 /*
1670 * Our emulated CPU doesn't support all the possible features. For the
1671 * sake of simplicity (and probably mental sanity), wipe out a number
1672 * of feature bits we don't intend to support for the time being.
1673 * This list should get updated as new features get added to the NV
1674 * support, and new extension to the architecture.
1675 */
limit_nv_id_reg(struct kvm * kvm,u32 reg,u64 val)1676 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
1677 {
1678 u64 orig_val = val;
1679
1680 switch (reg) {
1681 case SYS_ID_AA64ISAR1_EL1:
1682 /* Support everything but LS64 and Spec Invalidation */
1683 val &= ~(ID_AA64ISAR1_EL1_LS64 |
1684 ID_AA64ISAR1_EL1_SPECRES);
1685 break;
1686
1687 case SYS_ID_AA64PFR0_EL1:
1688 /* No RME, AMU, MPAM, or S-EL2 */
1689 val &= ~(ID_AA64PFR0_EL1_RME |
1690 ID_AA64PFR0_EL1_AMU |
1691 ID_AA64PFR0_EL1_MPAM |
1692 ID_AA64PFR0_EL1_SEL2 |
1693 ID_AA64PFR0_EL1_EL3 |
1694 ID_AA64PFR0_EL1_EL2 |
1695 ID_AA64PFR0_EL1_EL1 |
1696 ID_AA64PFR0_EL1_EL0);
1697 /* 64bit only at any EL */
1698 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP);
1699 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP);
1700 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP);
1701 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP);
1702 break;
1703
1704 case SYS_ID_AA64PFR1_EL1:
1705 /* Only support BTI, SSBS, CSV2_frac */
1706 val &= ~(ID_AA64PFR1_EL1_PFAR |
1707 ID_AA64PFR1_EL1_MTEX |
1708 ID_AA64PFR1_EL1_THE |
1709 ID_AA64PFR1_EL1_GCS |
1710 ID_AA64PFR1_EL1_MTE_frac |
1711 ID_AA64PFR1_EL1_NMI |
1712 ID_AA64PFR1_EL1_SME |
1713 ID_AA64PFR1_EL1_RES0 |
1714 ID_AA64PFR1_EL1_MPAM_frac |
1715 ID_AA64PFR1_EL1_MTE);
1716 break;
1717
1718 case SYS_ID_AA64PFR2_EL1:
1719 /* GICv5 is not yet supported for NV */
1720 val &= ~ID_AA64PFR2_EL1_GCIE;
1721 break;
1722
1723 case SYS_ID_AA64MMFR0_EL1:
1724 /* Hide ExS, Secure Memory */
1725 val &= ~(ID_AA64MMFR0_EL1_EXS |
1726 ID_AA64MMFR0_EL1_TGRAN4_2 |
1727 ID_AA64MMFR0_EL1_TGRAN16_2 |
1728 ID_AA64MMFR0_EL1_TGRAN64_2 |
1729 ID_AA64MMFR0_EL1_SNSMEM);
1730
1731 /* Hide CNTPOFF if present */
1732 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP);
1733
1734 /* Disallow unsupported S2 page sizes */
1735 switch (PAGE_SIZE) {
1736 case SZ_64K:
1737 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI);
1738 fallthrough;
1739 case SZ_16K:
1740 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI);
1741 fallthrough;
1742 case SZ_4K:
1743 /* Support everything */
1744 break;
1745 }
1746
1747 /*
1748 * Since we can't support a guest S2 page size smaller
1749 * than the host's own page size (due to KVM only
1750 * populating its own S2 using the kernel's page
1751 * size), advertise the limitation using FEAT_GTG.
1752 */
1753 switch (PAGE_SIZE) {
1754 case SZ_4K:
1755 if (_has_tgran_2(orig_val, 4))
1756 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP);
1757 fallthrough;
1758 case SZ_16K:
1759 if (_has_tgran_2(orig_val, 16))
1760 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP);
1761 fallthrough;
1762 case SZ_64K:
1763 if (_has_tgran_2(orig_val, 64))
1764 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP);
1765 break;
1766 }
1767
1768 /* Cap PARange to 48bits */
1769 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48);
1770 break;
1771
1772 case SYS_ID_AA64MMFR1_EL1:
1773 val &= ~(ID_AA64MMFR1_EL1_CMOW |
1774 ID_AA64MMFR1_EL1_nTLBPA |
1775 ID_AA64MMFR1_EL1_ETS);
1776
1777 /* FEAT_E2H0 implies no VHE */
1778 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))
1779 val &= ~ID_AA64MMFR1_EL1_VH;
1780
1781 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF);
1782 break;
1783
1784 case SYS_ID_AA64MMFR2_EL1:
1785 val &= ~(ID_AA64MMFR2_EL1_BBM |
1786 ID_AA64MMFR2_EL1_TTL |
1787 GENMASK_ULL(47, 44) |
1788 ID_AA64MMFR2_EL1_ST |
1789 ID_AA64MMFR2_EL1_CCIDX |
1790 ID_AA64MMFR2_EL1_VARange);
1791
1792 /* Force TTL support */
1793 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP);
1794 break;
1795
1796 case SYS_ID_AA64MMFR4_EL1:
1797 /*
1798 * You get EITHER
1799 *
1800 * - FEAT_VHE without FEAT_E2H0
1801 * - FEAT_NV limited to FEAT_NV2(p1)/NV3
1802 * - HCR_EL2.NV1 being RES0
1803 *
1804 * OR
1805 *
1806 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV
1807 *
1808 * Life is too short for anything else.
1809 */
1810 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) {
1811 val = 0;
1812 } else {
1813 val &= ID_AA64MMFR4_EL1_NV_frac;
1814 if (cpus_have_final_cap(ARM64_HAS_NV3))
1815 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3);
1816 else if (cpus_have_final_cap(ARM64_HAS_NV2P1))
1817 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1);
1818 else
1819 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY);
1820 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1);
1821 }
1822 break;
1823
1824 case SYS_ID_AA64DFR0_EL1:
1825 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */
1826 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff |
1827 ID_AA64DFR0_EL1_BRBE |
1828 ID_AA64DFR0_EL1_MTPMU |
1829 ID_AA64DFR0_EL1_TraceBuffer |
1830 ID_AA64DFR0_EL1_TraceFilt |
1831 ID_AA64DFR0_EL1_PMSVer |
1832 ID_AA64DFR0_EL1_CTX_CMPs |
1833 ID_AA64DFR0_EL1_SEBEP |
1834 ID_AA64DFR0_EL1_PMSS |
1835 ID_AA64DFR0_EL1_TraceVer);
1836
1837 /*
1838 * FEAT_Debugv8p9 requires support for extended breakpoints /
1839 * watchpoints.
1840 */
1841 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1842 break;
1843 }
1844
1845 return val;
1846 }
1847
kvm_vcpu_apply_reg_masks(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr,u64 v)1848 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu,
1849 enum vcpu_sysreg sr, u64 v)
1850 {
1851 struct resx resx;
1852
1853 resx = kvm_get_sysreg_resx(vcpu->kvm, sr);
1854 v &= ~resx.res0;
1855 v |= resx.res1;
1856
1857 return v;
1858 }
1859
set_sysreg_masks(struct kvm * kvm,int sr,struct resx resx)1860 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx)
1861 {
1862 BUILD_BUG_ON(!__builtin_constant_p(sr));
1863 BUILD_BUG_ON(sr < __SANITISED_REG_START__);
1864 BUILD_BUG_ON(sr >= NR_SYS_REGS);
1865
1866 kvm_set_sysreg_resx(kvm, sr, resx);
1867 }
1868
kvm_init_nv_sysregs(struct kvm_vcpu * vcpu)1869 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
1870 {
1871 struct kvm *kvm = vcpu->kvm;
1872 struct resx resx;
1873
1874 lockdep_assert_held(&kvm->arch.config_lock);
1875
1876 if (kvm->arch.sysreg_masks)
1877 goto out;
1878
1879 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks),
1880 GFP_KERNEL_ACCOUNT);
1881 if (!kvm->arch.sysreg_masks)
1882 return -ENOMEM;
1883
1884 /* VTTBR_EL2 */
1885 resx = (typeof(resx)){};
1886 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
1887 resx.res0 |= GENMASK(63, 56);
1888 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
1889 resx.res0 |= VTTBR_CNP_BIT;
1890 set_sysreg_masks(kvm, VTTBR_EL2, resx);
1891
1892 /* VTCR_EL2 */
1893 resx = get_reg_fixed_bits(kvm, VTCR_EL2);
1894 set_sysreg_masks(kvm, VTCR_EL2, resx);
1895
1896 /* VMPIDR_EL2 */
1897 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24);
1898 resx.res1 = BIT(31);
1899 set_sysreg_masks(kvm, VMPIDR_EL2, resx);
1900
1901 /* HCR_EL2 */
1902 resx = get_reg_fixed_bits(kvm, HCR_EL2);
1903 set_sysreg_masks(kvm, HCR_EL2, resx);
1904
1905 /* NVHCR_EL2 */
1906 resx = get_reg_fixed_bits(kvm, NVHCR_EL2);
1907 set_sysreg_masks(kvm, NVHCR_EL2, resx);
1908
1909 /* HCRX_EL2 */
1910 resx = get_reg_fixed_bits(kvm, HCRX_EL2);
1911 set_sysreg_masks(kvm, HCRX_EL2, resx);
1912
1913 /* HFG[RW]TR_EL2 */
1914 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2);
1915 set_sysreg_masks(kvm, HFGRTR_EL2, resx);
1916 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2);
1917 set_sysreg_masks(kvm, HFGWTR_EL2, resx);
1918
1919 /* HDFG[RW]TR_EL2 */
1920 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2);
1921 set_sysreg_masks(kvm, HDFGRTR_EL2, resx);
1922 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2);
1923 set_sysreg_masks(kvm, HDFGWTR_EL2, resx);
1924
1925 /* HFGITR_EL2 */
1926 resx = get_reg_fixed_bits(kvm, HFGITR_EL2);
1927 set_sysreg_masks(kvm, HFGITR_EL2, resx);
1928
1929 /* HAFGRTR_EL2 - not a lot to see here */
1930 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2);
1931 set_sysreg_masks(kvm, HAFGRTR_EL2, resx);
1932
1933 /* HFG[RW]TR2_EL2 */
1934 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2);
1935 set_sysreg_masks(kvm, HFGRTR2_EL2, resx);
1936 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2);
1937 set_sysreg_masks(kvm, HFGWTR2_EL2, resx);
1938
1939 /* HDFG[RW]TR2_EL2 */
1940 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2);
1941 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx);
1942 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2);
1943 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx);
1944
1945 /* HFGITR2_EL2 */
1946 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2);
1947 set_sysreg_masks(kvm, HFGITR2_EL2, resx);
1948
1949 /* TCR2_EL2 */
1950 resx = get_reg_fixed_bits(kvm, TCR2_EL2);
1951 set_sysreg_masks(kvm, TCR2_EL2, resx);
1952
1953 /* SCTLR_EL1 */
1954 resx = get_reg_fixed_bits(kvm, SCTLR_EL1);
1955 set_sysreg_masks(kvm, SCTLR_EL1, resx);
1956
1957 /* SCTLR_EL2 */
1958 resx = get_reg_fixed_bits(kvm, SCTLR_EL2);
1959 set_sysreg_masks(kvm, SCTLR_EL2, resx);
1960
1961 /* SCTLR2_ELx */
1962 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1);
1963 set_sysreg_masks(kvm, SCTLR2_EL1, resx);
1964 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2);
1965 set_sysreg_masks(kvm, SCTLR2_EL2, resx);
1966
1967 /* MDCR_EL2 */
1968 resx = get_reg_fixed_bits(kvm, MDCR_EL2);
1969 set_sysreg_masks(kvm, MDCR_EL2, resx);
1970
1971 /* CNTHCTL_EL2 */
1972 resx.res0 = GENMASK(63, 20);
1973 resx.res1 = 0;
1974 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP))
1975 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK;
1976 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) {
1977 resx.res0 |= CNTHCTL_ECV;
1978 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP))
1979 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT |
1980 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT);
1981 }
1982 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1983 resx.res0 |= GENMASK(11, 8);
1984 set_sysreg_masks(kvm, CNTHCTL_EL2, resx);
1985
1986 /* ICH_HCR_EL2 */
1987 resx.res0 = ICH_HCR_EL2_RES0;
1988 resx.res1 = ICH_HCR_EL2_RES1;
1989 if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS))
1990 resx.res0 |= ICH_HCR_EL2_TDIR;
1991 /* No GICv4 is presented to the guest */
1992 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;
1993 set_sysreg_masks(kvm, ICH_HCR_EL2, resx);
1994
1995 /* VNCR_EL2 */
1996 resx.res0 = VNCR_EL2_RES0;
1997 resx.res1 = VNCR_EL2_RES1;
1998 set_sysreg_masks(kvm, VNCR_EL2, resx);
1999
2000 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */
2001 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4);
2002 resx.res1 = ZCR_ELx_RES1;
2003 set_sysreg_masks(kvm, ZCR_EL2, resx);
2004
2005 out:
2006 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
2007 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0);
2008
2009 return 0;
2010 }
2011
check_nested_vcpu_requests(struct kvm_vcpu * vcpu)2012 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
2013 {
2014 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) {
2015 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
2016
2017 write_lock(&vcpu->kvm->mmu_lock);
2018 if (mmu->pending_unmap) {
2019 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true);
2020 mmu->pending_unmap = false;
2021 }
2022 write_unlock(&vcpu->kvm->mmu_lock);
2023 }
2024
2025 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
2026 kvm_map_l1_vncr(vcpu);
2027
2028 /* Must be last, as may switch context! */
2029 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
2030 kvm_inject_nested_irq(vcpu);
2031 }
2032
2033 /*
2034 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
2035 * can write to HCR_EL2 behind our back, potentially changing the exception
2036 * routing / masking for even the host context.
2037 *
2038 * What follows is some slop to (1) react to exception routing / masking and (2)
2039 * preserve the pending SError state across translation regimes.
2040 */
kvm_nested_flush_hwstate(struct kvm_vcpu * vcpu)2041 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
2042 {
2043 if (!vcpu_has_nv(vcpu))
2044 return;
2045
2046 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
2047 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
2048 }
2049
kvm_nested_sync_hwstate(struct kvm_vcpu * vcpu)2050 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
2051 {
2052 unsigned long *hcr = vcpu_hcr(vcpu);
2053
2054 if (!vcpu_has_nv(vcpu))
2055 return;
2056
2057 /*
2058 * We previously decided that an SError was deliverable to the guest.
2059 * Reap the pending state from HCR_EL2 and...
2060 */
2061 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
2062 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
2063
2064 /*
2065 * Re-attempt SError injection in case the deliverability has changed,
2066 * which is necessary to faithfully emulate WFI the case of a pending
2067 * SError being a wakeup condition.
2068 */
2069 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
2070 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
2071 }
2072
2073 /*
2074 * KVM unconditionally sets most of these traps anyway but use an allowlist
2075 * to document the guest hypervisor traps that may take precedence and guard
2076 * against future changes to the non-nested trap configuration.
2077 */
2078 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \
2079 MDCR_EL2_TDA | \
2080 MDCR_EL2_TDRA | \
2081 MDCR_EL2_TTRF | \
2082 MDCR_EL2_TPMS | \
2083 MDCR_EL2_TPM | \
2084 MDCR_EL2_TPMCR | \
2085 MDCR_EL2_TDCC | \
2086 MDCR_EL2_TDOSA)
2087
kvm_nested_setup_mdcr_el2(struct kvm_vcpu * vcpu)2088 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu)
2089 {
2090 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
2091
2092 if (is_nested_ctxt(vcpu))
2093 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
2094 /*
2095 * In yet another example where FEAT_NV2 is fscking broken, accesses
2096 * to MDSCR_EL1 are redirected to the VNCR despite having an effect
2097 * at EL2. Use a big hammer to apply sanity.
2098 *
2099 * Unless of course we have FEAT_FGT, in which case we can precisely
2100 * trap MDSCR_EL1.
2101 */
2102 else if (!cpus_have_final_cap(ARM64_HAS_FGT))
2103 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
2104 }
2105