1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4 * Author: Jintack Lim <jintack.lim@linaro.org>
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10
11 #include <asm/fixmap.h>
12 #include <asm/kvm_arm.h>
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_mmu.h>
15 #include <asm/kvm_nested.h>
16 #include <asm/sysreg.h>
17
18 #include "sys_regs.h"
19
20 struct vncr_tlb {
21 /* The guest's VNCR_EL2 */
22 u64 gva;
23 struct s1_walk_info wi;
24 struct s1_walk_result wr;
25
26 u64 hpa;
27 bool hpa_writable;
28
29 /* -1 when not mapped on a CPU */
30 int cpu;
31
32 /*
33 * true if the TLB is valid. Can only be changed with the
34 * mmu_lock held.
35 */
36 bool valid;
37 };
38
39 /*
40 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
41 * memory usage and potential number of different sets of S2 PTs in
42 * the guests. Running out of S2 MMUs only affects performance (we
43 * will invalidate them more often).
44 */
45 #define S2_MMU_PER_VCPU 2
46
kvm_init_nested(struct kvm * kvm)47 void kvm_init_nested(struct kvm *kvm)
48 {
49 kvm->arch.nested_mmus = NULL;
50 kvm->arch.nested_mmus_size = 0;
51 atomic_set(&kvm->arch.vncr_map_count, 0);
52 }
53
init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)54 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
55 {
56 /*
57 * We only initialise the IPA range on the canonical MMU, which
58 * defines the contract between KVM and userspace on where the
59 * "hardware" is in the IPA space. This affects the validity of MMIO
60 * exits forwarded to userspace, for example.
61 *
62 * For nested S2s, we use the PARange as exposed to the guest, as it
63 * is allowed to use it at will to expose whatever memory map it
64 * wants to its own guests as it would be on real HW.
65 */
66 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
67 }
68
kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)69 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
70 {
71 struct kvm *kvm = vcpu->kvm;
72 struct kvm_s2_mmu *tmp;
73 int num_mmus, ret = 0;
74
75 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&
76 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
77 return -EINVAL;
78
79 if (!vcpu->arch.ctxt.vncr_array)
80 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
81 __GFP_ZERO);
82
83 if (!vcpu->arch.ctxt.vncr_array)
84 return -ENOMEM;
85
86 /*
87 * Let's treat memory allocation failures as benign: If we fail to
88 * allocate anything, return an error and keep the allocated array
89 * alive. Userspace may try to recover by initializing the vcpu
90 * again, and there is no reason to affect the whole VM for this.
91 */
92 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
93
94 if (num_mmus > kvm->arch.nested_mmus_size) {
95 tmp = kvcalloc(num_mmus, sizeof(*tmp), GFP_KERNEL_ACCOUNT);
96 if (!tmp)
97 return -ENOMEM;
98
99 write_lock(&kvm->mmu_lock);
100
101 if (kvm->arch.nested_mmus_size) {
102 memcpy(tmp, kvm->arch.nested_mmus,
103 size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size));
104
105 for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
106 tmp[i].pgt->mmu = &tmp[i];
107 }
108
109 swap(kvm->arch.nested_mmus, tmp);
110
111 write_unlock(&kvm->mmu_lock);
112
113 kvfree(tmp);
114 }
115
116 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
117 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
118
119 if (ret) {
120 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
121 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
122
123 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
124 vcpu->arch.ctxt.vncr_array = NULL;
125
126 return ret;
127 }
128
129 kvm->arch.nested_mmus_size = num_mmus;
130
131 return 0;
132 }
133
134 struct s2_walk_info {
135 u64 baddr;
136 unsigned int max_oa_bits;
137 unsigned int pgshift;
138 unsigned int sl;
139 unsigned int t0sz;
140 bool be;
141 bool ha;
142 };
143
compute_fsc(int level,u32 fsc)144 static u32 compute_fsc(int level, u32 fsc)
145 {
146 return fsc | (level & 0x3);
147 }
148
esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)149 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
150 {
151 u32 esr;
152
153 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
154 esr |= compute_fsc(level, fsc);
155 return esr;
156 }
157
get_ia_size(struct s2_walk_info * wi)158 static int get_ia_size(struct s2_walk_info *wi)
159 {
160 return 64 - wi->t0sz;
161 }
162
check_base_s2_limits(struct kvm_vcpu * vcpu,struct s2_walk_info * wi,int level,int input_size,int stride)163 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
164 int level, int input_size, int stride)
165 {
166 int start_size, pa_max;
167
168 pa_max = kvm_get_pa_bits(vcpu->kvm);
169
170 /* Check translation limits */
171 switch (BIT(wi->pgshift)) {
172 case SZ_64K:
173 if (level == 0 || (level == 1 && pa_max <= 42))
174 return -EFAULT;
175 break;
176 case SZ_16K:
177 if (level == 0 || (level == 1 && pa_max <= 40))
178 return -EFAULT;
179 break;
180 case SZ_4K:
181 if (level < 0 || (level == 0 && pa_max <= 42))
182 return -EFAULT;
183 break;
184 }
185
186 /* Check input size limits */
187 if (input_size > pa_max)
188 return -EFAULT;
189
190 /* Check number of entries in starting level table */
191 start_size = input_size - ((3 - level) * stride + wi->pgshift);
192 if (start_size < 1 || start_size > stride + 4)
193 return -EFAULT;
194
195 return 0;
196 }
197
198 /* Check if output is within boundaries */
check_output_size(struct s2_walk_info * wi,phys_addr_t output)199 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
200 {
201 unsigned int output_size = wi->max_oa_bits;
202
203 if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
204 return -1;
205
206 return 0;
207 }
208
read_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 * desc,struct s2_walk_info * wi)209 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc,
210 struct s2_walk_info *wi)
211 {
212 u64 val;
213 int r;
214
215 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
216 if (r)
217 return r;
218
219 /*
220 * Handle reversedescriptors if endianness differs between the
221 * host and the guest hypervisor.
222 */
223 if (wi->be)
224 *desc = be64_to_cpu((__force __be64)val);
225 else
226 *desc = le64_to_cpu((__force __le64)val);
227
228 return 0;
229 }
230
swap_guest_s2_desc(struct kvm_vcpu * vcpu,phys_addr_t pa,u64 old,u64 new,struct s2_walk_info * wi)231 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,
232 struct s2_walk_info *wi)
233 {
234 if (wi->be) {
235 old = (__force u64)cpu_to_be64(old);
236 new = (__force u64)cpu_to_be64(new);
237 } else {
238 old = (__force u64)cpu_to_le64(old);
239 new = (__force u64)cpu_to_le64(new);
240 }
241
242 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
243 }
244
245 /*
246 * This is essentially a C-version of the pseudo code from the ARM ARM
247 * AArch64.TranslationTableWalk function. I strongly recommend looking at
248 * that pseudocode in trying to understand this.
249 *
250 * Must be called with the kvm->srcu read lock held
251 */
walk_nested_s2_pgd(struct kvm_vcpu * vcpu,phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)252 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
253 struct s2_walk_info *wi, struct kvm_s2_trans *out)
254 {
255 int first_block_level, level, stride, input_size, base_lower_bound;
256 phys_addr_t base_addr;
257 unsigned int addr_top, addr_bottom;
258 u64 desc, new_desc; /* page table entry */
259 int ret;
260 phys_addr_t paddr;
261
262 switch (BIT(wi->pgshift)) {
263 default:
264 case SZ_64K:
265 case SZ_16K:
266 level = 3 - wi->sl;
267 first_block_level = 2;
268 break;
269 case SZ_4K:
270 level = 2 - wi->sl;
271 first_block_level = 1;
272 break;
273 }
274
275 stride = wi->pgshift - 3;
276 input_size = get_ia_size(wi);
277 if (input_size > 48 || input_size < 25)
278 return -EFAULT;
279
280 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);
281 if (WARN_ON(ret)) {
282 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT);
283 return ret;
284 }
285
286 base_lower_bound = 3 + input_size - ((3 - level) * stride +
287 wi->pgshift);
288 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
289
290 if (check_output_size(wi, base_addr)) {
291 /* R_BFHQH */
292 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ);
293 return 1;
294 }
295
296 addr_top = input_size - 1;
297
298 while (1) {
299 phys_addr_t index;
300
301 addr_bottom = (3 - level) * stride + wi->pgshift;
302 index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
303 >> (addr_bottom - 3);
304
305 paddr = base_addr | index;
306 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);
307 if (ret < 0) {
308 out->esr = ESR_ELx_FSC_SEA_TTW(level);
309 return ret;
310 }
311
312 new_desc = desc;
313
314 /* Check for valid descriptor at this point */
315 if (!(desc & KVM_PTE_VALID)) {
316 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
317 out->desc = desc;
318 return 1;
319 }
320
321 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) {
322 if (level < 3)
323 break;
324
325 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
326 out->desc = desc;
327 return 1;
328 }
329
330 /* We're at the final level */
331 if (level == 3)
332 break;
333
334 if (check_output_size(wi, desc)) {
335 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
336 out->desc = desc;
337 return 1;
338 }
339
340 base_addr = desc & GENMASK_ULL(47, wi->pgshift);
341
342 level += 1;
343 addr_top = addr_bottom - 1;
344 }
345
346 if (level < first_block_level) {
347 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
348 out->desc = desc;
349 return 1;
350 }
351
352 if (check_output_size(wi, desc)) {
353 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
354 out->desc = desc;
355 return 1;
356 }
357
358 if (wi->ha)
359 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
360
361 if (new_desc != desc) {
362 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);
363 if (ret == -EAGAIN)
364 return ret;
365 if (ret) {
366 out->esr = ESR_ELx_FSC_SEA_TTW(level);
367 out->desc = desc;
368 return 1;
369 }
370
371 desc = new_desc;
372 }
373
374 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) {
375 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
376 out->desc = desc;
377 return 1;
378 }
379
380 addr_bottom += contiguous_bit_shift(desc, wi, level);
381
382 /* Calculate and return the result */
383 paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
384 (ipa & GENMASK_ULL(addr_bottom - 1, 0));
385 out->output = paddr;
386 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
387 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
388 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
389 out->level = level;
390 out->desc = desc;
391 return 0;
392 }
393
394 #define _has_tgran_2(__r, __sz) \
395 ({ \
396 u64 _s1, _s2, _mmfr0 = __r; \
397 \
398 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
399 TGRAN##__sz##_2, _mmfr0); \
400 \
401 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \
402 TGRAN##__sz, _mmfr0); \
403 \
404 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \
405 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \
406 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \
407 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \
408 })
409
has_tgran_2(u64 mmfr0,unsigned int shift)410 static bool has_tgran_2(u64 mmfr0, unsigned int shift)
411 {
412 switch (shift) {
413 case 12:
414 return _has_tgran_2(mmfr0, 4);
415 case 14:
416 return _has_tgran_2(mmfr0, 16);
417 case 16:
418 return _has_tgran_2(mmfr0, 64);
419 default:
420 BUG();
421 }
422 }
423
fallback_tgran2_shift(u64 mmfr0)424 static unsigned int fallback_tgran2_shift(u64 mmfr0)
425 {
426 if (has_tgran_2(mmfr0, PAGE_SHIFT))
427 return PAGE_SHIFT;
428 else if (has_tgran_2(mmfr0, 12))
429 return 12;
430 else if (has_tgran_2(mmfr0, 14))
431 return 14;
432 else if (has_tgran_2(mmfr0, 16))
433 return 16;
434 else
435 return PAGE_SHIFT;
436 }
437
vtcr_to_tg0_pgshift(struct kvm * kvm,u64 vtcr)438 static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr)
439 {
440 u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr);
441 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
442 unsigned int shift;
443
444 switch (tg0) {
445 case VTCR_EL2_TG0_4K:
446 shift = 12;
447 break;
448 case VTCR_EL2_TG0_16K:
449 shift = 14;
450 break;
451 case VTCR_EL2_TG0_64K:
452 /* IMPDEF: treat any other value as 64k, subject to fallback */
453 default:
454 shift = 16;
455 }
456
457 /*
458 * If TGx is programmed to an unimplemented value (not advertised in
459 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is
460 * written, as per the architecture. Choose an available one while
461 * prioritizing PAGE_SIZE.
462 */
463 if (!has_tgran_2(mmfr0, shift))
464 return fallback_tgran2_shift(mmfr0);
465
466 return shift;
467 }
468
vtcr_to_tg0_pgsize(struct kvm * kvm,u64 vtcr)469 static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr)
470 {
471 return BIT(vtcr_to_tg0_pgshift(kvm, vtcr));
472 }
473
setup_s2_walk(struct kvm_vcpu * vcpu,struct s2_walk_info * wi)474 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi)
475 {
476 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
477
478 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
479 wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK;
480 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);
481 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
482 /* Global limit for now, should eventually be per-VM */
483 wi->max_oa_bits = min(get_kvm_ipa_limit(),
484 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));
485 wi->ha = vtcr & VTCR_EL2_HA;
486 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
487 }
488
kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)489 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
490 struct kvm_s2_trans *result)
491 {
492 struct s2_walk_info wi;
493 int ret;
494
495 result->esr = 0;
496
497 if (!vcpu_has_nv(vcpu))
498 return 0;
499
500 setup_s2_walk(vcpu, &wi);
501
502 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
503 if (ret)
504 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
505
506 return ret;
507 }
508
ttl_to_size(u8 ttl)509 static unsigned int ttl_to_size(u8 ttl)
510 {
511 int level = ttl & 3;
512 int gran = (ttl >> 2) & 3;
513 unsigned int max_size = 0;
514
515 switch (gran) {
516 case TLBI_TTL_TG_4K:
517 switch (level) {
518 case 0:
519 break;
520 case 1:
521 max_size = SZ_1G;
522 break;
523 case 2:
524 max_size = SZ_2M;
525 break;
526 case 3:
527 max_size = SZ_4K;
528 break;
529 }
530 break;
531 case TLBI_TTL_TG_16K:
532 switch (level) {
533 case 0:
534 case 1:
535 break;
536 case 2:
537 max_size = SZ_32M;
538 break;
539 case 3:
540 max_size = SZ_16K;
541 break;
542 }
543 break;
544 case TLBI_TTL_TG_64K:
545 switch (level) {
546 case 0:
547 case 1:
548 /* No 52bit IPA support */
549 break;
550 case 2:
551 max_size = SZ_512M;
552 break;
553 case 3:
554 max_size = SZ_64K;
555 break;
556 }
557 break;
558 default: /* No size information */
559 break;
560 }
561
562 return max_size;
563 }
564
pgshift_level_to_ttl(u16 shift,u8 level)565 static u8 pgshift_level_to_ttl(u16 shift, u8 level)
566 {
567 u8 ttl;
568
569 switch(shift) {
570 case 12:
571 ttl = TLBI_TTL_TG_4K;
572 break;
573 case 14:
574 ttl = TLBI_TTL_TG_16K;
575 break;
576 case 16:
577 ttl = TLBI_TTL_TG_64K;
578 break;
579 default:
580 BUG();
581 }
582
583 ttl <<= 2;
584 ttl |= level & 3;
585
586 return ttl;
587 }
588
589 /*
590 * Compute the equivalent of the TTL field by parsing the shadow PT. The
591 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
592 * retrieved from first entry carrying the level as a tag.
593 */
get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)594 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
595 {
596 size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr);
597 u64 tmp, sz = 0;
598 kvm_pte_t pte;
599 u8 ttl, level;
600
601 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
602
603 switch (tg0_size) {
604 case SZ_4K:
605 ttl = (TLBI_TTL_TG_4K << 2);
606 break;
607 case SZ_16K:
608 ttl = (TLBI_TTL_TG_16K << 2);
609 break;
610 case SZ_64K:
611 default: /* IMPDEF: treat any other value as 64k */
612 ttl = (TLBI_TTL_TG_64K << 2);
613 break;
614 }
615
616 tmp = addr;
617
618 again:
619 /* Iteratively compute the block sizes for a particular granule size */
620 switch (tg0_size) {
621 case SZ_4K:
622 if (sz < SZ_4K) sz = SZ_4K;
623 else if (sz < SZ_2M) sz = SZ_2M;
624 else if (sz < SZ_1G) sz = SZ_1G;
625 else sz = 0;
626 break;
627 case SZ_16K:
628 if (sz < SZ_16K) sz = SZ_16K;
629 else if (sz < SZ_32M) sz = SZ_32M;
630 else sz = 0;
631 break;
632 case SZ_64K:
633 default: /* IMPDEF: treat any other value as 64k */
634 if (sz < SZ_64K) sz = SZ_64K;
635 else if (sz < SZ_512M) sz = SZ_512M;
636 else sz = 0;
637 break;
638 }
639
640 if (sz == 0)
641 return 0;
642
643 tmp &= ~(sz - 1);
644 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
645 goto again;
646 if (!(pte & PTE_VALID))
647 goto again;
648 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
649 if (!level)
650 goto again;
651
652 ttl |= level;
653
654 /*
655 * We now have found some level information in the shadow S2. Check
656 * that the resulting range is actually including the original IPA.
657 */
658 sz = ttl_to_size(ttl);
659 if (addr < (tmp + sz))
660 return ttl;
661
662 return 0;
663 }
664
compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)665 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
666 {
667 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
668 unsigned long max_size;
669 u8 ttl;
670
671 ttl = FIELD_GET(TLBI_TTL_MASK, val);
672
673 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
674 /* No TTL, check the shadow S2 for a hint */
675 u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
676 ttl = get_guest_mapping_ttl(mmu, addr);
677 }
678
679 max_size = ttl_to_size(ttl);
680
681 if (!max_size) {
682 /* Compute the maximum extent of the invalidation */
683 switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) {
684 case SZ_4K:
685 max_size = SZ_1G;
686 break;
687 case SZ_16K:
688 max_size = SZ_32M;
689 break;
690 case SZ_64K:
691 default: /* IMPDEF: treat any other value as 64k */
692 /*
693 * No, we do not support 52bit IPA in nested yet. Once
694 * we do, this should be 4TB.
695 */
696 max_size = SZ_512M;
697 break;
698 }
699 }
700
701 WARN_ON(!max_size);
702 return max_size;
703 }
704
705 /*
706 * We can have multiple *different* MMU contexts with the same VMID:
707 *
708 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
709 *
710 * - Multiple vcpus using private S2s (huh huh...), hence differing by the
711 * VBBTR_EL2.BADDR address
712 *
713 * - A combination of the above...
714 *
715 * We can always identify which MMU context to pick at run-time. However,
716 * TLB invalidation involving a VMID must take action on all the TLBs using
717 * this particular VMID. This translates into applying the same invalidation
718 * operation to all the contexts that are using this VMID. Moar phun!
719 */
kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))720 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
721 const union tlbi_info *info,
722 void (*tlbi_callback)(struct kvm_s2_mmu *,
723 const union tlbi_info *))
724 {
725 write_lock(&kvm->mmu_lock);
726
727 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
728 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
729
730 if (!kvm_s2_mmu_valid(mmu))
731 continue;
732
733 if (vmid == get_vmid(mmu->tlb_vttbr))
734 tlbi_callback(mmu, info);
735 }
736
737 write_unlock(&kvm->mmu_lock);
738 }
739
lookup_s2_mmu(struct kvm_vcpu * vcpu)740 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
741 {
742 struct kvm *kvm = vcpu->kvm;
743 bool nested_stage2_enabled;
744 u64 vttbr, vtcr, hcr;
745
746 lockdep_assert_held_write(&kvm->mmu_lock);
747
748 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
749 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
750 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
751
752 nested_stage2_enabled = hcr & HCR_VM;
753
754 /* Don't consider the CnP bit for the vttbr match */
755 vttbr &= ~VTTBR_CNP_BIT;
756
757 /*
758 * Two possibilities when looking up a S2 MMU context:
759 *
760 * - either S2 is enabled in the guest, and we need a context that is
761 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
762 * which makes it safe from a TLB conflict perspective (a broken
763 * guest won't be able to generate them),
764 *
765 * - or S2 is disabled, and we need a context that is S2-disabled
766 * and matches the VMID only, as all TLBs are tagged by VMID even
767 * if S2 translation is disabled.
768 */
769 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
770 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
771
772 if (!kvm_s2_mmu_valid(mmu))
773 continue;
774
775 if (nested_stage2_enabled &&
776 mmu->nested_stage2_enabled &&
777 vttbr == mmu->tlb_vttbr &&
778 vtcr == mmu->tlb_vtcr)
779 return mmu;
780
781 if (!nested_stage2_enabled &&
782 !mmu->nested_stage2_enabled &&
783 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
784 return mmu;
785 }
786 return NULL;
787 }
788
get_s2_mmu_nested(struct kvm_vcpu * vcpu)789 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
790 {
791 struct kvm *kvm = vcpu->kvm;
792 struct kvm_s2_mmu *s2_mmu;
793 int i;
794
795 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
796
797 s2_mmu = lookup_s2_mmu(vcpu);
798 if (s2_mmu)
799 goto out;
800
801 /*
802 * Make sure we don't always search from the same point, or we
803 * will always reuse a potentially active context, leaving
804 * free contexts unused.
805 */
806 for (i = kvm->arch.nested_mmus_next;
807 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
808 i++) {
809 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
810
811 if (atomic_read(&s2_mmu->refcnt) == 0)
812 break;
813 }
814 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
815
816 /* Set the scene for the next search */
817 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
818
819 /* Make sure we don't forget to do the laundry */
820 if (kvm_s2_mmu_valid(s2_mmu)) {
821 kvm_nested_s2_ptdump_remove_debugfs(s2_mmu);
822 s2_mmu->pending_unmap = true;
823 }
824
825 /*
826 * The virtual VMID (modulo CnP) will be used as a key when matching
827 * an existing kvm_s2_mmu.
828 *
829 * We cache VTCR at allocation time, once and for all. It'd be great
830 * if the guest didn't screw that one up, as this is not very
831 * forgiving...
832 */
833 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
834 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
835 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
836
837 kvm_nested_s2_ptdump_create_debugfs(s2_mmu);
838
839 out:
840 atomic_inc(&s2_mmu->refcnt);
841
842 /*
843 * Set the vCPU request to perform an unmap, even if the pending unmap
844 * originates from another vCPU. This guarantees that the MMU has been
845 * completely unmapped before any vCPU actually uses it, and allows
846 * multiple vCPUs to lend a hand with completing the unmap.
847 */
848 if (s2_mmu->pending_unmap)
849 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu);
850
851 return s2_mmu;
852 }
853
kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)854 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
855 {
856 /* CnP being set denotes an invalid entry */
857 mmu->tlb_vttbr = VTTBR_CNP_BIT;
858 mmu->nested_stage2_enabled = false;
859 atomic_set(&mmu->refcnt, 0);
860 }
861
kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)862 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
863 {
864 /*
865 * If the vCPU kept its reference on the MMU after the last put,
866 * keep rolling with it.
867 */
868 if (is_hyp_ctxt(vcpu)) {
869 if (!vcpu->arch.hw_mmu)
870 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
871 } else {
872 if (!vcpu->arch.hw_mmu) {
873 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
874 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
875 }
876
877 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
878 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
879 }
880 }
881
this_cpu_reset_vncr_fixmap(struct kvm_vcpu * vcpu)882 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu)
883 {
884 if (!host_data_test_flag(L1_VNCR_MAPPED))
885 return;
886
887 BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id());
888 BUG_ON(is_hyp_ctxt(vcpu));
889
890 clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu));
891 vcpu->arch.vncr_tlb->cpu = -1;
892 host_data_clear_flag(L1_VNCR_MAPPED);
893 atomic_dec(&vcpu->kvm->arch.vncr_map_count);
894 }
895
kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)896 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
897 {
898 /* Unconditionally drop the VNCR mapping if we have one */
899 this_cpu_reset_vncr_fixmap(vcpu);
900
901 /*
902 * Keep a reference on the associated stage-2 MMU if the vCPU is
903 * scheduling out and not in WFI emulation, suggesting it is likely to
904 * reuse the MMU sometime soon.
905 */
906 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI))
907 return;
908
909 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
910 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
911
912 vcpu->arch.hw_mmu = NULL;
913 }
914
915 /*
916 * Returns non-zero if permission fault is handled by injecting it to the next
917 * level hypervisor.
918 */
kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)919 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
920 {
921 bool forward_fault = false;
922
923 trans->esr = 0;
924
925 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
926 return 0;
927
928 if (kvm_vcpu_trap_is_iabt(vcpu)) {
929 if (vcpu_mode_priv(vcpu))
930 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
931 else
932 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
933 } else {
934 bool write_fault = kvm_is_write_fault(vcpu);
935
936 forward_fault = ((write_fault && !trans->writable) ||
937 (!write_fault && !trans->readable));
938 }
939
940 if (forward_fault)
941 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
942
943 return forward_fault;
944 }
945
kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)946 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
947 {
948 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
949 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
950
951 return kvm_inject_nested_sync(vcpu, esr_el2);
952 }
953
get_asid_by_regime(struct kvm_vcpu * vcpu,enum trans_regime regime)954 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime)
955 {
956 enum vcpu_sysreg ttbr_elx;
957 u64 tcr;
958 u16 asid;
959
960 switch (regime) {
961 case TR_EL10:
962 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
963 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1;
964 break;
965 case TR_EL20:
966 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
967 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2;
968 break;
969 default:
970 BUG();
971 }
972
973 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx));
974 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
975 !(tcr & TCR_ASID16))
976 asid &= GENMASK(7, 0);
977
978 return asid;
979 }
980
invalidate_vncr(struct vncr_tlb * vt)981 static void invalidate_vncr(struct vncr_tlb *vt)
982 {
983 vt->valid = false;
984 if (vt->cpu != -1)
985 clear_fixmap(vncr_fixmap(vt->cpu));
986 }
987
988 /*
989 * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and
990 * either can race against a vcpu not being onlined yet (no pseudo-TLB
991 * allocated). Similarly, the TLB might be invalid. Skip those, as they
992 * obviously don't participate in the invalidation at this stage.
993 */
994 #define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \
995 kvm_for_each_vcpu(idx, vcpup, kvm) \
996 if (((tlbp) = vcpup->arch.vncr_tlb) && \
997 (tlbp)->valid)
998
kvm_invalidate_vncr_ipa(struct kvm * kvm,u64 start,u64 end)999 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
1000 {
1001 struct kvm_vcpu *vcpu;
1002 struct vncr_tlb *vt;
1003 unsigned long i;
1004
1005 lockdep_assert_held_write(&kvm->mmu_lock);
1006
1007 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1008 return;
1009
1010 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
1011 u64 ipa_start, ipa_end, ipa_size;
1012
1013 ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
1014 vt->wr.level));
1015 ipa_start = vt->wr.pa & ~(ipa_size - 1);
1016 ipa_end = ipa_start + ipa_size;
1017
1018 if (ipa_end <= start || ipa_start >= end)
1019 continue;
1020
1021 invalidate_vncr(vt);
1022 }
1023 }
1024
1025 struct s1e2_tlbi_scope {
1026 enum {
1027 TLBI_ALL,
1028 TLBI_VA,
1029 TLBI_VAA,
1030 TLBI_ASID,
1031 } type;
1032
1033 u16 asid;
1034 u64 va;
1035 u64 size;
1036 };
1037
invalidate_vncr_va(struct kvm * kvm,struct s1e2_tlbi_scope * scope)1038 static void invalidate_vncr_va(struct kvm *kvm,
1039 struct s1e2_tlbi_scope *scope)
1040 {
1041 struct kvm_vcpu *vcpu;
1042 struct vncr_tlb *vt;
1043 unsigned long i;
1044
1045 lockdep_assert_held_write(&kvm->mmu_lock);
1046
1047 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
1048 u64 va_start, va_end, va_size;
1049
1050 va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
1051 vt->wr.level));
1052 va_start = vt->gva & ~(va_size - 1);
1053 va_end = va_start + va_size;
1054
1055 switch (scope->type) {
1056 case TLBI_ALL:
1057 break;
1058
1059 case TLBI_VA:
1060 if (va_end <= scope->va ||
1061 va_start >= (scope->va + scope->size))
1062 continue;
1063 if (vt->wr.nG && vt->wr.asid != scope->asid)
1064 continue;
1065 break;
1066
1067 case TLBI_VAA:
1068 if (va_end <= scope->va ||
1069 va_start >= (scope->va + scope->size))
1070 continue;
1071 break;
1072
1073 case TLBI_ASID:
1074 if (!vt->wr.nG || vt->wr.asid != scope->asid)
1075 continue;
1076 break;
1077 }
1078
1079 invalidate_vncr(vt);
1080 }
1081 }
1082
1083 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48)
1084
compute_s1_tlbi_range(struct kvm_vcpu * vcpu,u32 inst,u64 val,struct s1e2_tlbi_scope * scope)1085 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
1086 struct s1e2_tlbi_scope *scope)
1087 {
1088 switch (inst) {
1089 case OP_TLBI_ALLE2:
1090 case OP_TLBI_ALLE2IS:
1091 case OP_TLBI_ALLE2OS:
1092 case OP_TLBI_VMALLE1:
1093 case OP_TLBI_VMALLE1IS:
1094 case OP_TLBI_VMALLE1OS:
1095 case OP_TLBI_ALLE2NXS:
1096 case OP_TLBI_ALLE2ISNXS:
1097 case OP_TLBI_ALLE2OSNXS:
1098 case OP_TLBI_VMALLE1NXS:
1099 case OP_TLBI_VMALLE1ISNXS:
1100 case OP_TLBI_VMALLE1OSNXS:
1101 scope->type = TLBI_ALL;
1102 break;
1103 case OP_TLBI_VAE2:
1104 case OP_TLBI_VAE2IS:
1105 case OP_TLBI_VAE2OS:
1106 case OP_TLBI_VAE1:
1107 case OP_TLBI_VAE1IS:
1108 case OP_TLBI_VAE1OS:
1109 case OP_TLBI_VAE2NXS:
1110 case OP_TLBI_VAE2ISNXS:
1111 case OP_TLBI_VAE2OSNXS:
1112 case OP_TLBI_VAE1NXS:
1113 case OP_TLBI_VAE1ISNXS:
1114 case OP_TLBI_VAE1OSNXS:
1115 case OP_TLBI_VALE2:
1116 case OP_TLBI_VALE2IS:
1117 case OP_TLBI_VALE2OS:
1118 case OP_TLBI_VALE1:
1119 case OP_TLBI_VALE1IS:
1120 case OP_TLBI_VALE1OS:
1121 case OP_TLBI_VALE2NXS:
1122 case OP_TLBI_VALE2ISNXS:
1123 case OP_TLBI_VALE2OSNXS:
1124 case OP_TLBI_VALE1NXS:
1125 case OP_TLBI_VALE1ISNXS:
1126 case OP_TLBI_VALE1OSNXS:
1127 scope->type = TLBI_VA;
1128 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1129 if (!scope->size)
1130 scope->size = SZ_1G;
1131 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1132 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1133 break;
1134 case OP_TLBI_ASIDE1:
1135 case OP_TLBI_ASIDE1IS:
1136 case OP_TLBI_ASIDE1OS:
1137 case OP_TLBI_ASIDE1NXS:
1138 case OP_TLBI_ASIDE1ISNXS:
1139 case OP_TLBI_ASIDE1OSNXS:
1140 scope->type = TLBI_ASID;
1141 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
1142 break;
1143 case OP_TLBI_VAAE1:
1144 case OP_TLBI_VAAE1IS:
1145 case OP_TLBI_VAAE1OS:
1146 case OP_TLBI_VAAE1NXS:
1147 case OP_TLBI_VAAE1ISNXS:
1148 case OP_TLBI_VAAE1OSNXS:
1149 case OP_TLBI_VAALE1:
1150 case OP_TLBI_VAALE1IS:
1151 case OP_TLBI_VAALE1OS:
1152 case OP_TLBI_VAALE1NXS:
1153 case OP_TLBI_VAALE1ISNXS:
1154 case OP_TLBI_VAALE1OSNXS:
1155 scope->type = TLBI_VAA;
1156 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
1157 if (!scope->size)
1158 scope->size = SZ_1G;
1159 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1);
1160 break;
1161 case OP_TLBI_RVAE2:
1162 case OP_TLBI_RVAE2IS:
1163 case OP_TLBI_RVAE2OS:
1164 case OP_TLBI_RVAE1:
1165 case OP_TLBI_RVAE1IS:
1166 case OP_TLBI_RVAE1OS:
1167 case OP_TLBI_RVAE2NXS:
1168 case OP_TLBI_RVAE2ISNXS:
1169 case OP_TLBI_RVAE2OSNXS:
1170 case OP_TLBI_RVAE1NXS:
1171 case OP_TLBI_RVAE1ISNXS:
1172 case OP_TLBI_RVAE1OSNXS:
1173 case OP_TLBI_RVALE2:
1174 case OP_TLBI_RVALE2IS:
1175 case OP_TLBI_RVALE2OS:
1176 case OP_TLBI_RVALE1:
1177 case OP_TLBI_RVALE1IS:
1178 case OP_TLBI_RVALE1OS:
1179 case OP_TLBI_RVALE2NXS:
1180 case OP_TLBI_RVALE2ISNXS:
1181 case OP_TLBI_RVALE2OSNXS:
1182 case OP_TLBI_RVALE1NXS:
1183 case OP_TLBI_RVALE1ISNXS:
1184 case OP_TLBI_RVALE1OSNXS:
1185 scope->type = TLBI_VA;
1186 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid);
1187 break;
1188 case OP_TLBI_RVAAE1:
1189 case OP_TLBI_RVAAE1IS:
1190 case OP_TLBI_RVAAE1OS:
1191 case OP_TLBI_RVAAE1NXS:
1192 case OP_TLBI_RVAAE1ISNXS:
1193 case OP_TLBI_RVAAE1OSNXS:
1194 case OP_TLBI_RVAALE1:
1195 case OP_TLBI_RVAALE1IS:
1196 case OP_TLBI_RVAALE1OS:
1197 case OP_TLBI_RVAALE1NXS:
1198 case OP_TLBI_RVAALE1ISNXS:
1199 case OP_TLBI_RVAALE1OSNXS:
1200 scope->type = TLBI_VAA;
1201 scope->va = decode_range_tlbi(val, &scope->size, NULL);
1202 break;
1203 }
1204 }
1205
kvm_handle_s1e2_tlbi(struct kvm_vcpu * vcpu,u32 inst,u64 val)1206 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1207 {
1208 struct s1e2_tlbi_scope scope = {};
1209
1210 compute_s1_tlbi_range(vcpu, inst, val, &scope);
1211
1212 guard(write_lock)(&vcpu->kvm->mmu_lock);
1213 invalidate_vncr_va(vcpu->kvm, &scope);
1214 }
1215
kvm_nested_s2_wp(struct kvm * kvm)1216 void kvm_nested_s2_wp(struct kvm *kvm)
1217 {
1218 int i;
1219
1220 lockdep_assert_held_write(&kvm->mmu_lock);
1221
1222 if (!kvm->arch.nested_mmus_size)
1223 return;
1224
1225 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1226 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1227
1228 if (kvm_s2_mmu_valid(mmu))
1229 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
1230 }
1231
1232 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1233 }
1234
kvm_nested_s2_unmap(struct kvm * kvm,bool may_block)1235 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
1236 {
1237 int i;
1238
1239 lockdep_assert_held_write(&kvm->mmu_lock);
1240
1241 if (!kvm->arch.nested_mmus_size)
1242 return;
1243
1244 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1245 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1246
1247 if (kvm_s2_mmu_valid(mmu))
1248 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
1249 }
1250
1251 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
1252 }
1253
kvm_nested_s2_flush(struct kvm * kvm)1254 void kvm_nested_s2_flush(struct kvm *kvm)
1255 {
1256 int i;
1257
1258 lockdep_assert_held_write(&kvm->mmu_lock);
1259
1260 if (!kvm->arch.nested_mmus_size)
1261 return;
1262
1263 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1264 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1265
1266 if (kvm_s2_mmu_valid(mmu))
1267 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
1268 }
1269 }
1270
kvm_arch_flush_shadow_all(struct kvm * kvm)1271 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1272 {
1273 int i;
1274
1275 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
1276 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
1277
1278 if (!WARN_ON(atomic_read(&mmu->refcnt)))
1279 kvm_free_stage2_pgd(mmu);
1280 }
1281 kvfree(kvm->arch.nested_mmus);
1282 kvm->arch.nested_mmus = NULL;
1283 kvm->arch.nested_mmus_size = 0;
1284 kvm_uninit_stage2_mmu(kvm);
1285 }
1286
1287 /*
1288 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter:
1289 *
1290 * - We introduce an internal representation of a vcpu-private TLB,
1291 * representing the mapping between the guest VA contained in VNCR_EL2,
1292 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at.
1293 *
1294 * - On translation fault from a nested VNCR access, we create such a TLB.
1295 * If there is no mapping to describe, the guest inherits the fault.
1296 * Crucially, no actual mapping is done at this stage.
1297 *
1298 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above
1299 * TLB exists, we map it in the fixmap for this CPU, and run with it. We
1300 * have to respect the permissions dictated by the guest, but not the
1301 * memory type (FWB is a must).
1302 *
1303 * - Note that we usually don't do a vcpu_load() on the back of a fault
1304 * (unless we are preempted), so the resolution of a translation fault
1305 * must go via a request that will map the VNCR page in the fixmap.
1306 * vcpu_load() might as well use the same mechanism.
1307 *
1308 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was
1309 * mapped, we unmap it. Yes it is that simple. The TLB still exists
1310 * though, and may be reused at a later load.
1311 *
1312 * - On permission fault, we simply forward the fault to the guest's EL2.
1313 * Get out of my way.
1314 *
1315 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that
1316 * intersects with the TLBI request, invalidate it, and unmap the page
1317 * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1318 * this requires some wide-ranging locking to ensure that nothing races
1319 * against it. This may require some refcounting to avoid the search when
1320 * no such TLB is present.
1321 *
1322 * - On MMU notifiers, we must invalidate our TLB in a similar way, but
1323 * looking at the IPA instead. The funny part is that there may not be a
1324 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST
1325 * instructions.
1326 */
1327
kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu * vcpu)1328 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1329 {
1330 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1331 return 0;
1332
1333 if (!vcpu->arch.vncr_tlb) {
1334 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb,
1335 GFP_KERNEL_ACCOUNT);
1336
1337 /*
1338 * Taking the lock on assignment ensures that the TLB is
1339 * seen as initialised when following the pointer (release
1340 * semantics of the unlock), and avoids having acquires on
1341 * each user which already take the lock.
1342 */
1343 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
1344 vcpu->arch.vncr_tlb = vt;
1345 }
1346
1347 if (!vcpu->arch.vncr_tlb)
1348 return -ENOMEM;
1349
1350 return 0;
1351 }
1352
read_vncr_el2(struct kvm_vcpu * vcpu)1353 static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1354 {
1355 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1356 }
1357
kvm_translate_vncr(struct kvm_vcpu * vcpu,bool * is_gmem)1358 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
1359 {
1360 struct kvm_memory_slot *memslot;
1361 bool write_fault, writable;
1362 unsigned long mmu_seq;
1363 struct vncr_tlb *vt;
1364 struct page *page;
1365 u64 va, pfn, gfn;
1366 int ret;
1367
1368 vt = vcpu->arch.vncr_tlb;
1369
1370 /*
1371 * If we're about to walk the EL2 S1 PTs, we must invalidate the
1372 * current TLB, as it could be sampled from another vcpu doing a
1373 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single
1374 * translation, so not much of a choice.
1375 *
1376 * We also prepare the next walk wilst we're at it.
1377 */
1378 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1379 this_cpu_reset_vncr_fixmap(vcpu);
1380 vt->valid = false;
1381
1382 vt->wi = (struct s1_walk_info) {
1383 .regime = TR_EL20,
1384 .as_el0 = false,
1385 .pan = false,
1386 };
1387 vt->wr = (struct s1_walk_result){};
1388 }
1389
1390 guard(srcu)(&vcpu->kvm->srcu);
1391
1392 va = read_vncr_el2(vcpu);
1393
1394 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1395 if (ret)
1396 return ret;
1397
1398 write_fault = kvm_is_write_fault(vcpu);
1399
1400 mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1401 smp_rmb();
1402
1403 gfn = vt->wr.pa >> PAGE_SHIFT;
1404 memslot = gfn_to_memslot(vcpu->kvm, gfn);
1405 if (!memslot) {
1406 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1407 return -EFAULT;
1408 }
1409
1410 *is_gmem = kvm_slot_has_gmem(memslot);
1411 if (!*is_gmem) {
1412 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0,
1413 &writable, &page);
1414 if (is_error_noslot_pfn(pfn)) {
1415 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1416 return -EFAULT;
1417 }
1418 } else {
1419 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
1420 if (ret) {
1421 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
1422 write_fault, false, false);
1423 return ret;
1424 }
1425
1426 writable = !(memslot->flags & KVM_MEM_READONLY);
1427 }
1428
1429 /*
1430 * FIXME: This check is too restrictive as KVM allows cacheable memory
1431 * attributes for PFNMAP VMAs that have cacheable attributes in host
1432 * stage-1.
1433 */
1434 if (!pfn_is_map_memory(pfn)) {
1435 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1436 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
1437 return -EINVAL;
1438 }
1439
1440 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1441 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
1442 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1443 return -EAGAIN;
1444 }
1445
1446 vt->gva = va;
1447 vt->hpa = pfn << PAGE_SHIFT;
1448 vt->hpa_writable = writable;
1449 vt->valid = true;
1450 vt->cpu = -1;
1451
1452 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1453 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
1454 }
1455
1456 if (vt->wr.pw && vt->hpa_writable)
1457 mark_page_dirty(vcpu->kvm, gfn);
1458
1459 return 0;
1460 }
1461
handle_vncr_perm(struct kvm_vcpu * vcpu)1462 static void handle_vncr_perm(struct kvm_vcpu *vcpu)
1463 {
1464 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1465 u64 esr = kvm_vcpu_get_esr(vcpu);
1466 u64 fsc;
1467
1468 /*
1469 * Promote to an external abort if the stage-1 permits writes but the
1470 * HPA is read-only (e.g. RO memslot).
1471 */
1472 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable)
1473 fsc = ESR_ELx_FSC_EXTABT;
1474 /*
1475 * Otherwise, inject a permission fault using the guest's translation
1476 * level rather than the host's.
1477 */
1478 else
1479 fsc = ESR_ELx_FSC_PERM_L(vt->wr.level);
1480
1481 esr &= ~ESR_ELx_FSC;
1482 esr |= FIELD_PREP(ESR_ELx_FSC, fsc);
1483
1484 kvm_inject_nested_sync(vcpu, esr);
1485 }
1486
kvm_handle_vncr_abort(struct kvm_vcpu * vcpu)1487 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1488 {
1489 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1490 u64 esr = kvm_vcpu_get_esr(vcpu);
1491 bool is_gmem = false;
1492 bool perm;
1493 int ret;
1494
1495 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR));
1496
1497 if (kvm_vcpu_abt_issea(vcpu))
1498 return kvm_handle_guest_sea(vcpu);
1499
1500 if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) {
1501 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr);
1502 return -EIO;
1503 }
1504
1505 ret = kvm_translate_vncr(vcpu, &is_gmem);
1506 switch (ret) {
1507 case -EAGAIN:
1508 /* Let's try again... */
1509 return 1;
1510 case -ENOMEM:
1511 /*
1512 * For guest_memfd, this indicates that it failed to
1513 * create a folio to back the memory. Inform userspace.
1514 */
1515 if (is_gmem)
1516 return 0;
1517 /* Otherwise, let's try again... */
1518 break;
1519 case -EFAULT:
1520 case -EIO:
1521 case -EHWPOISON:
1522 if (is_gmem)
1523 return 0;
1524 fallthrough;
1525 case -EINVAL:
1526 case -ENOENT:
1527 case -EACCES:
1528 /*
1529 * Translation failed, inject the corresponding
1530 * exception back to EL2.
1531 */
1532 esr &= ~ESR_ELx_FSC;
1533 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
1534
1535 kvm_inject_nested_sync(vcpu, esr);
1536 break;
1537 case 0:
1538 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr;
1539 if (!perm)
1540 handle_vncr_perm(vcpu);
1541 break;
1542 }
1543
1544 return 1;
1545 }
1546
kvm_map_l1_vncr(struct kvm_vcpu * vcpu)1547 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1548 {
1549 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1550 pgprot_t prot;
1551
1552 guard(preempt)();
1553 guard(read_lock)(&vcpu->kvm->mmu_lock);
1554
1555 /*
1556 * The request to map VNCR may have raced against some other
1557 * event, such as an interrupt, and may not be valid anymore.
1558 */
1559 if (is_hyp_ctxt(vcpu))
1560 return;
1561
1562 /*
1563 * Check that the pseudo-TLB is valid and that VNCR_EL2 still
1564 * contains the expected value. If it doesn't, we simply bail out
1565 * without a mapping -- a transformed MSR/MRS will generate the
1566 * fault and allows us to populate the pseudo-TLB.
1567 */
1568 if (!vt->valid)
1569 return;
1570
1571 if (read_vncr_el2(vcpu) != vt->gva)
1572 return;
1573
1574 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid)
1575 return;
1576
1577 vt->cpu = smp_processor_id();
1578
1579 if (vt->hpa_writable && vt->wr.pw && vt->wr.pr)
1580 prot = PAGE_KERNEL;
1581 else if (vt->wr.pr)
1582 prot = PAGE_KERNEL_RO;
1583 else
1584 prot = PAGE_NONE;
1585
1586 /*
1587 * We can't map write-only (or no permission at all) in the kernel,
1588 * but the guest can do it if using POE, so we'll have to turn a
1589 * translation fault into a permission fault at runtime.
1590 * FIXME: WO doesn't work at all, need POE support in the kernel.
1591 */
1592 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
1593 __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot);
1594 host_data_set_flag(L1_VNCR_MAPPED);
1595 atomic_inc(&vcpu->kvm->arch.vncr_map_count);
1596 }
1597 }
1598
1599 /*
1600 * Our emulated CPU doesn't support all the possible features. For the
1601 * sake of simplicity (and probably mental sanity), wipe out a number
1602 * of feature bits we don't intend to support for the time being.
1603 * This list should get updated as new features get added to the NV
1604 * support, and new extension to the architecture.
1605 */
limit_nv_id_reg(struct kvm * kvm,u32 reg,u64 val)1606 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
1607 {
1608 u64 orig_val = val;
1609
1610 switch (reg) {
1611 case SYS_ID_AA64ISAR1_EL1:
1612 /* Support everything but LS64 and Spec Invalidation */
1613 val &= ~(ID_AA64ISAR1_EL1_LS64 |
1614 ID_AA64ISAR1_EL1_SPECRES);
1615 break;
1616
1617 case SYS_ID_AA64PFR0_EL1:
1618 /* No RME, AMU, MPAM, or S-EL2 */
1619 val &= ~(ID_AA64PFR0_EL1_RME |
1620 ID_AA64PFR0_EL1_AMU |
1621 ID_AA64PFR0_EL1_MPAM |
1622 ID_AA64PFR0_EL1_SEL2 |
1623 ID_AA64PFR0_EL1_EL3 |
1624 ID_AA64PFR0_EL1_EL2 |
1625 ID_AA64PFR0_EL1_EL1 |
1626 ID_AA64PFR0_EL1_EL0);
1627 /* 64bit only at any EL */
1628 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP);
1629 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP);
1630 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP);
1631 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP);
1632 break;
1633
1634 case SYS_ID_AA64PFR1_EL1:
1635 /* Only support BTI, SSBS, CSV2_frac */
1636 val &= ~(ID_AA64PFR1_EL1_PFAR |
1637 ID_AA64PFR1_EL1_MTEX |
1638 ID_AA64PFR1_EL1_THE |
1639 ID_AA64PFR1_EL1_GCS |
1640 ID_AA64PFR1_EL1_MTE_frac |
1641 ID_AA64PFR1_EL1_NMI |
1642 ID_AA64PFR1_EL1_SME |
1643 ID_AA64PFR1_EL1_RES0 |
1644 ID_AA64PFR1_EL1_MPAM_frac |
1645 ID_AA64PFR1_EL1_MTE);
1646 break;
1647
1648 case SYS_ID_AA64PFR2_EL1:
1649 /* GICv5 is not yet supported for NV */
1650 val &= ~ID_AA64PFR2_EL1_GCIE;
1651 break;
1652
1653 case SYS_ID_AA64MMFR0_EL1:
1654 /* Hide ExS, Secure Memory */
1655 val &= ~(ID_AA64MMFR0_EL1_EXS |
1656 ID_AA64MMFR0_EL1_TGRAN4_2 |
1657 ID_AA64MMFR0_EL1_TGRAN16_2 |
1658 ID_AA64MMFR0_EL1_TGRAN64_2 |
1659 ID_AA64MMFR0_EL1_SNSMEM);
1660
1661 /* Hide CNTPOFF if present */
1662 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP);
1663
1664 /* Disallow unsupported S2 page sizes */
1665 switch (PAGE_SIZE) {
1666 case SZ_64K:
1667 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI);
1668 fallthrough;
1669 case SZ_16K:
1670 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI);
1671 fallthrough;
1672 case SZ_4K:
1673 /* Support everything */
1674 break;
1675 }
1676
1677 /*
1678 * Since we can't support a guest S2 page size smaller
1679 * than the host's own page size (due to KVM only
1680 * populating its own S2 using the kernel's page
1681 * size), advertise the limitation using FEAT_GTG.
1682 */
1683 switch (PAGE_SIZE) {
1684 case SZ_4K:
1685 if (_has_tgran_2(orig_val, 4))
1686 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP);
1687 fallthrough;
1688 case SZ_16K:
1689 if (_has_tgran_2(orig_val, 16))
1690 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP);
1691 fallthrough;
1692 case SZ_64K:
1693 if (_has_tgran_2(orig_val, 64))
1694 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP);
1695 break;
1696 }
1697
1698 /* Cap PARange to 48bits */
1699 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48);
1700 break;
1701
1702 case SYS_ID_AA64MMFR1_EL1:
1703 val &= ~(ID_AA64MMFR1_EL1_CMOW |
1704 ID_AA64MMFR1_EL1_nTLBPA |
1705 ID_AA64MMFR1_EL1_ETS);
1706
1707 /* FEAT_E2H0 implies no VHE */
1708 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))
1709 val &= ~ID_AA64MMFR1_EL1_VH;
1710
1711 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF);
1712 break;
1713
1714 case SYS_ID_AA64MMFR2_EL1:
1715 val &= ~(ID_AA64MMFR2_EL1_BBM |
1716 ID_AA64MMFR2_EL1_TTL |
1717 GENMASK_ULL(47, 44) |
1718 ID_AA64MMFR2_EL1_ST |
1719 ID_AA64MMFR2_EL1_CCIDX |
1720 ID_AA64MMFR2_EL1_VARange);
1721
1722 /* Force TTL support */
1723 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP);
1724 break;
1725
1726 case SYS_ID_AA64MMFR4_EL1:
1727 /*
1728 * You get EITHER
1729 *
1730 * - FEAT_VHE without FEAT_E2H0
1731 * - FEAT_NV limited to FEAT_NV2
1732 * - HCR_EL2.NV1 being RES0
1733 *
1734 * OR
1735 *
1736 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV
1737 *
1738 * Life is too short for anything else.
1739 */
1740 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) {
1741 val = 0;
1742 } else {
1743 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY);
1744 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1);
1745 }
1746 break;
1747
1748 case SYS_ID_AA64DFR0_EL1:
1749 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */
1750 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff |
1751 ID_AA64DFR0_EL1_BRBE |
1752 ID_AA64DFR0_EL1_MTPMU |
1753 ID_AA64DFR0_EL1_TraceBuffer |
1754 ID_AA64DFR0_EL1_TraceFilt |
1755 ID_AA64DFR0_EL1_PMSVer |
1756 ID_AA64DFR0_EL1_CTX_CMPs |
1757 ID_AA64DFR0_EL1_SEBEP |
1758 ID_AA64DFR0_EL1_PMSS |
1759 ID_AA64DFR0_EL1_TraceVer);
1760
1761 /*
1762 * FEAT_Debugv8p9 requires support for extended breakpoints /
1763 * watchpoints.
1764 */
1765 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1766 break;
1767 }
1768
1769 return val;
1770 }
1771
kvm_vcpu_apply_reg_masks(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr,u64 v)1772 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu,
1773 enum vcpu_sysreg sr, u64 v)
1774 {
1775 struct resx resx;
1776
1777 resx = kvm_get_sysreg_resx(vcpu->kvm, sr);
1778 v &= ~resx.res0;
1779 v |= resx.res1;
1780
1781 return v;
1782 }
1783
set_sysreg_masks(struct kvm * kvm,int sr,struct resx resx)1784 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx)
1785 {
1786 BUILD_BUG_ON(!__builtin_constant_p(sr));
1787 BUILD_BUG_ON(sr < __SANITISED_REG_START__);
1788 BUILD_BUG_ON(sr >= NR_SYS_REGS);
1789
1790 kvm_set_sysreg_resx(kvm, sr, resx);
1791 }
1792
kvm_init_nv_sysregs(struct kvm_vcpu * vcpu)1793 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
1794 {
1795 struct kvm *kvm = vcpu->kvm;
1796 struct resx resx;
1797
1798 lockdep_assert_held(&kvm->arch.config_lock);
1799
1800 if (kvm->arch.sysreg_masks)
1801 goto out;
1802
1803 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks),
1804 GFP_KERNEL_ACCOUNT);
1805 if (!kvm->arch.sysreg_masks)
1806 return -ENOMEM;
1807
1808 /* VTTBR_EL2 */
1809 resx = (typeof(resx)){};
1810 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
1811 resx.res0 |= GENMASK(63, 56);
1812 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
1813 resx.res0 |= VTTBR_CNP_BIT;
1814 set_sysreg_masks(kvm, VTTBR_EL2, resx);
1815
1816 /* VTCR_EL2 */
1817 resx = get_reg_fixed_bits(kvm, VTCR_EL2);
1818 set_sysreg_masks(kvm, VTCR_EL2, resx);
1819
1820 /* VMPIDR_EL2 */
1821 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24);
1822 resx.res1 = BIT(31);
1823 set_sysreg_masks(kvm, VMPIDR_EL2, resx);
1824
1825 /* HCR_EL2 */
1826 resx = get_reg_fixed_bits(kvm, HCR_EL2);
1827 set_sysreg_masks(kvm, HCR_EL2, resx);
1828
1829 /* HCRX_EL2 */
1830 resx = get_reg_fixed_bits(kvm, HCRX_EL2);
1831 set_sysreg_masks(kvm, HCRX_EL2, resx);
1832
1833 /* HFG[RW]TR_EL2 */
1834 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2);
1835 set_sysreg_masks(kvm, HFGRTR_EL2, resx);
1836 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2);
1837 set_sysreg_masks(kvm, HFGWTR_EL2, resx);
1838
1839 /* HDFG[RW]TR_EL2 */
1840 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2);
1841 set_sysreg_masks(kvm, HDFGRTR_EL2, resx);
1842 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2);
1843 set_sysreg_masks(kvm, HDFGWTR_EL2, resx);
1844
1845 /* HFGITR_EL2 */
1846 resx = get_reg_fixed_bits(kvm, HFGITR_EL2);
1847 set_sysreg_masks(kvm, HFGITR_EL2, resx);
1848
1849 /* HAFGRTR_EL2 - not a lot to see here */
1850 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2);
1851 set_sysreg_masks(kvm, HAFGRTR_EL2, resx);
1852
1853 /* HFG[RW]TR2_EL2 */
1854 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2);
1855 set_sysreg_masks(kvm, HFGRTR2_EL2, resx);
1856 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2);
1857 set_sysreg_masks(kvm, HFGWTR2_EL2, resx);
1858
1859 /* HDFG[RW]TR2_EL2 */
1860 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2);
1861 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx);
1862 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2);
1863 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx);
1864
1865 /* HFGITR2_EL2 */
1866 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2);
1867 set_sysreg_masks(kvm, HFGITR2_EL2, resx);
1868
1869 /* TCR2_EL2 */
1870 resx = get_reg_fixed_bits(kvm, TCR2_EL2);
1871 set_sysreg_masks(kvm, TCR2_EL2, resx);
1872
1873 /* SCTLR_EL1 */
1874 resx = get_reg_fixed_bits(kvm, SCTLR_EL1);
1875 set_sysreg_masks(kvm, SCTLR_EL1, resx);
1876
1877 /* SCTLR_EL2 */
1878 resx = get_reg_fixed_bits(kvm, SCTLR_EL2);
1879 set_sysreg_masks(kvm, SCTLR_EL2, resx);
1880
1881 /* SCTLR2_ELx */
1882 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1);
1883 set_sysreg_masks(kvm, SCTLR2_EL1, resx);
1884 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2);
1885 set_sysreg_masks(kvm, SCTLR2_EL2, resx);
1886
1887 /* MDCR_EL2 */
1888 resx = get_reg_fixed_bits(kvm, MDCR_EL2);
1889 set_sysreg_masks(kvm, MDCR_EL2, resx);
1890
1891 /* CNTHCTL_EL2 */
1892 resx.res0 = GENMASK(63, 20);
1893 resx.res1 = 0;
1894 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP))
1895 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK;
1896 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) {
1897 resx.res0 |= CNTHCTL_ECV;
1898 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP))
1899 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT |
1900 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT);
1901 }
1902 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
1903 resx.res0 |= GENMASK(11, 8);
1904 set_sysreg_masks(kvm, CNTHCTL_EL2, resx);
1905
1906 /* ICH_HCR_EL2 */
1907 resx.res0 = ICH_HCR_EL2_RES0;
1908 resx.res1 = ICH_HCR_EL2_RES1;
1909 if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS))
1910 resx.res0 |= ICH_HCR_EL2_TDIR;
1911 /* No GICv4 is presented to the guest */
1912 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;
1913 set_sysreg_masks(kvm, ICH_HCR_EL2, resx);
1914
1915 /* VNCR_EL2 */
1916 resx.res0 = VNCR_EL2_RES0;
1917 resx.res1 = VNCR_EL2_RES1;
1918 set_sysreg_masks(kvm, VNCR_EL2, resx);
1919
1920 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */
1921 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4);
1922 resx.res1 = ZCR_ELx_RES1;
1923 set_sysreg_masks(kvm, ZCR_EL2, resx);
1924
1925 out:
1926 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
1927 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0);
1928
1929 return 0;
1930 }
1931
check_nested_vcpu_requests(struct kvm_vcpu * vcpu)1932 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
1933 {
1934 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) {
1935 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
1936
1937 write_lock(&vcpu->kvm->mmu_lock);
1938 if (mmu->pending_unmap) {
1939 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true);
1940 mmu->pending_unmap = false;
1941 }
1942 write_unlock(&vcpu->kvm->mmu_lock);
1943 }
1944
1945 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
1946 kvm_map_l1_vncr(vcpu);
1947
1948 /* Must be last, as may switch context! */
1949 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
1950 kvm_inject_nested_irq(vcpu);
1951 }
1952
1953 /*
1954 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
1955 * can write to HCR_EL2 behind our back, potentially changing the exception
1956 * routing / masking for even the host context.
1957 *
1958 * What follows is some slop to (1) react to exception routing / masking and (2)
1959 * preserve the pending SError state across translation regimes.
1960 */
kvm_nested_flush_hwstate(struct kvm_vcpu * vcpu)1961 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
1962 {
1963 if (!vcpu_has_nv(vcpu))
1964 return;
1965
1966 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1967 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1968 }
1969
kvm_nested_sync_hwstate(struct kvm_vcpu * vcpu)1970 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
1971 {
1972 unsigned long *hcr = vcpu_hcr(vcpu);
1973
1974 if (!vcpu_has_nv(vcpu))
1975 return;
1976
1977 /*
1978 * We previously decided that an SError was deliverable to the guest.
1979 * Reap the pending state from HCR_EL2 and...
1980 */
1981 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
1982 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
1983
1984 /*
1985 * Re-attempt SError injection in case the deliverability has changed,
1986 * which is necessary to faithfully emulate WFI the case of a pending
1987 * SError being a wakeup condition.
1988 */
1989 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
1990 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
1991 }
1992
1993 /*
1994 * KVM unconditionally sets most of these traps anyway but use an allowlist
1995 * to document the guest hypervisor traps that may take precedence and guard
1996 * against future changes to the non-nested trap configuration.
1997 */
1998 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \
1999 MDCR_EL2_TDA | \
2000 MDCR_EL2_TDRA | \
2001 MDCR_EL2_TTRF | \
2002 MDCR_EL2_TPMS | \
2003 MDCR_EL2_TPM | \
2004 MDCR_EL2_TPMCR | \
2005 MDCR_EL2_TDCC | \
2006 MDCR_EL2_TDOSA)
2007
kvm_nested_setup_mdcr_el2(struct kvm_vcpu * vcpu)2008 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu)
2009 {
2010 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
2011
2012 if (is_nested_ctxt(vcpu))
2013 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
2014 /*
2015 * In yet another example where FEAT_NV2 is fscking broken, accesses
2016 * to MDSCR_EL1 are redirected to the VNCR despite having an effect
2017 * at EL2. Use a big hammer to apply sanity.
2018 *
2019 * Unless of course we have FEAT_FGT, in which case we can precisely
2020 * trap MDSCR_EL1.
2021 */
2022 else if (!cpus_have_final_cap(ARM64_HAS_FGT))
2023 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
2024 }
2025