1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 - Columbia University and Linaro Ltd.
4 * Author: Jintack Lim <jintack.lim@linaro.org>
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10
11 #include <asm/kvm_arm.h>
12 #include <asm/kvm_emulate.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/kvm_nested.h>
15 #include <asm/sysreg.h>
16
17 #include "sys_regs.h"
18
19 /* Protection against the sysreg repainting madness... */
20 #define NV_FTR(r, f) ID_AA64##r##_EL1_##f
21
22 /*
23 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
24 * memory usage and potential number of different sets of S2 PTs in
25 * the guests. Running out of S2 MMUs only affects performance (we
26 * will invalidate them more often).
27 */
28 #define S2_MMU_PER_VCPU 2
29
kvm_init_nested(struct kvm * kvm)30 void kvm_init_nested(struct kvm *kvm)
31 {
32 kvm->arch.nested_mmus = NULL;
33 kvm->arch.nested_mmus_size = 0;
34 }
35
init_nested_s2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu)36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
37 {
38 /*
39 * We only initialise the IPA range on the canonical MMU, which
40 * defines the contract between KVM and userspace on where the
41 * "hardware" is in the IPA space. This affects the validity of MMIO
42 * exits forwarded to userspace, for example.
43 *
44 * For nested S2s, we use the PARange as exposed to the guest, as it
45 * is allowed to use it at will to expose whatever memory map it
46 * wants to its own guests as it would be on real HW.
47 */
48 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
49 }
50
kvm_vcpu_init_nested(struct kvm_vcpu * vcpu)51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
52 {
53 struct kvm *kvm = vcpu->kvm;
54 struct kvm_s2_mmu *tmp;
55 int num_mmus, ret = 0;
56
57 /*
58 * Let's treat memory allocation failures as benign: If we fail to
59 * allocate anything, return an error and keep the allocated array
60 * alive. Userspace may try to recover by intializing the vcpu
61 * again, and there is no reason to affect the whole VM for this.
62 */
63 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
64 tmp = kvrealloc(kvm->arch.nested_mmus,
65 size_mul(sizeof(*kvm->arch.nested_mmus), kvm->arch.nested_mmus_size),
66 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus),
67 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
68 if (!tmp)
69 return -ENOMEM;
70
71 /*
72 * If we went through a realocation, adjust the MMU back-pointers in
73 * the previously initialised kvm_pgtable structures.
74 */
75 if (kvm->arch.nested_mmus != tmp)
76 for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
77 tmp[i].pgt->mmu = &tmp[i];
78
79 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
80 ret = init_nested_s2_mmu(kvm, &tmp[i]);
81
82 if (ret) {
83 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
84 kvm_free_stage2_pgd(&tmp[i]);
85
86 return ret;
87 }
88
89 kvm->arch.nested_mmus_size = num_mmus;
90 kvm->arch.nested_mmus = tmp;
91
92 return 0;
93 }
94
95 struct s2_walk_info {
96 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data);
97 void *data;
98 u64 baddr;
99 unsigned int max_oa_bits;
100 unsigned int pgshift;
101 unsigned int sl;
102 unsigned int t0sz;
103 bool be;
104 };
105
compute_fsc(int level,u32 fsc)106 static u32 compute_fsc(int level, u32 fsc)
107 {
108 return fsc | (level & 0x3);
109 }
110
esr_s2_fault(struct kvm_vcpu * vcpu,int level,u32 fsc)111 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
112 {
113 u32 esr;
114
115 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
116 esr |= compute_fsc(level, fsc);
117 return esr;
118 }
119
get_ia_size(struct s2_walk_info * wi)120 static int get_ia_size(struct s2_walk_info *wi)
121 {
122 return 64 - wi->t0sz;
123 }
124
check_base_s2_limits(struct s2_walk_info * wi,int level,int input_size,int stride)125 static int check_base_s2_limits(struct s2_walk_info *wi,
126 int level, int input_size, int stride)
127 {
128 int start_size, ia_size;
129
130 ia_size = get_ia_size(wi);
131
132 /* Check translation limits */
133 switch (BIT(wi->pgshift)) {
134 case SZ_64K:
135 if (level == 0 || (level == 1 && ia_size <= 42))
136 return -EFAULT;
137 break;
138 case SZ_16K:
139 if (level == 0 || (level == 1 && ia_size <= 40))
140 return -EFAULT;
141 break;
142 case SZ_4K:
143 if (level < 0 || (level == 0 && ia_size <= 42))
144 return -EFAULT;
145 break;
146 }
147
148 /* Check input size limits */
149 if (input_size > ia_size)
150 return -EFAULT;
151
152 /* Check number of entries in starting level table */
153 start_size = input_size - ((3 - level) * stride + wi->pgshift);
154 if (start_size < 1 || start_size > stride + 4)
155 return -EFAULT;
156
157 return 0;
158 }
159
160 /* Check if output is within boundaries */
check_output_size(struct s2_walk_info * wi,phys_addr_t output)161 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output)
162 {
163 unsigned int output_size = wi->max_oa_bits;
164
165 if (output_size != 48 && (output & GENMASK_ULL(47, output_size)))
166 return -1;
167
168 return 0;
169 }
170
171 /*
172 * This is essentially a C-version of the pseudo code from the ARM ARM
173 * AArch64.TranslationTableWalk function. I strongly recommend looking at
174 * that pseudocode in trying to understand this.
175 *
176 * Must be called with the kvm->srcu read lock held
177 */
walk_nested_s2_pgd(phys_addr_t ipa,struct s2_walk_info * wi,struct kvm_s2_trans * out)178 static int walk_nested_s2_pgd(phys_addr_t ipa,
179 struct s2_walk_info *wi, struct kvm_s2_trans *out)
180 {
181 int first_block_level, level, stride, input_size, base_lower_bound;
182 phys_addr_t base_addr;
183 unsigned int addr_top, addr_bottom;
184 u64 desc; /* page table entry */
185 int ret;
186 phys_addr_t paddr;
187
188 switch (BIT(wi->pgshift)) {
189 default:
190 case SZ_64K:
191 case SZ_16K:
192 level = 3 - wi->sl;
193 first_block_level = 2;
194 break;
195 case SZ_4K:
196 level = 2 - wi->sl;
197 first_block_level = 1;
198 break;
199 }
200
201 stride = wi->pgshift - 3;
202 input_size = get_ia_size(wi);
203 if (input_size > 48 || input_size < 25)
204 return -EFAULT;
205
206 ret = check_base_s2_limits(wi, level, input_size, stride);
207 if (WARN_ON(ret))
208 return ret;
209
210 base_lower_bound = 3 + input_size - ((3 - level) * stride +
211 wi->pgshift);
212 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound);
213
214 if (check_output_size(wi, base_addr)) {
215 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
216 return 1;
217 }
218
219 addr_top = input_size - 1;
220
221 while (1) {
222 phys_addr_t index;
223
224 addr_bottom = (3 - level) * stride + wi->pgshift;
225 index = (ipa & GENMASK_ULL(addr_top, addr_bottom))
226 >> (addr_bottom - 3);
227
228 paddr = base_addr | index;
229 ret = wi->read_desc(paddr, &desc, wi->data);
230 if (ret < 0)
231 return ret;
232
233 /*
234 * Handle reversedescriptors if endianness differs between the
235 * host and the guest hypervisor.
236 */
237 if (wi->be)
238 desc = be64_to_cpu((__force __be64)desc);
239 else
240 desc = le64_to_cpu((__force __le64)desc);
241
242 /* Check for valid descriptor at this point */
243 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) {
244 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
245 out->desc = desc;
246 return 1;
247 }
248
249 /* We're at the final level or block translation level */
250 if ((desc & 3) == 1 || level == 3)
251 break;
252
253 if (check_output_size(wi, desc)) {
254 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
255 out->desc = desc;
256 return 1;
257 }
258
259 base_addr = desc & GENMASK_ULL(47, wi->pgshift);
260
261 level += 1;
262 addr_top = addr_bottom - 1;
263 }
264
265 if (level < first_block_level) {
266 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
267 out->desc = desc;
268 return 1;
269 }
270
271 if (check_output_size(wi, desc)) {
272 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ);
273 out->desc = desc;
274 return 1;
275 }
276
277 if (!(desc & BIT(10))) {
278 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
279 out->desc = desc;
280 return 1;
281 }
282
283 addr_bottom += contiguous_bit_shift(desc, wi, level);
284
285 /* Calculate and return the result */
286 paddr = (desc & GENMASK_ULL(47, addr_bottom)) |
287 (ipa & GENMASK_ULL(addr_bottom - 1, 0));
288 out->output = paddr;
289 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
290 out->readable = desc & (0b01 << 6);
291 out->writable = desc & (0b10 << 6);
292 out->level = level;
293 out->desc = desc;
294 return 0;
295 }
296
read_guest_s2_desc(phys_addr_t pa,u64 * desc,void * data)297 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data)
298 {
299 struct kvm_vcpu *vcpu = data;
300
301 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc));
302 }
303
vtcr_to_walk_info(u64 vtcr,struct s2_walk_info * wi)304 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
305 {
306 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
307
308 switch (vtcr & VTCR_EL2_TG0_MASK) {
309 case VTCR_EL2_TG0_4K:
310 wi->pgshift = 12; break;
311 case VTCR_EL2_TG0_16K:
312 wi->pgshift = 14; break;
313 case VTCR_EL2_TG0_64K:
314 default: /* IMPDEF: treat any other value as 64k */
315 wi->pgshift = 16; break;
316 }
317
318 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
319 /* Global limit for now, should eventually be per-VM */
320 wi->max_oa_bits = min(get_kvm_ipa_limit(),
321 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr)));
322 }
323
kvm_walk_nested_s2(struct kvm_vcpu * vcpu,phys_addr_t gipa,struct kvm_s2_trans * result)324 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
325 struct kvm_s2_trans *result)
326 {
327 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
328 struct s2_walk_info wi;
329 int ret;
330
331 result->esr = 0;
332
333 if (!vcpu_has_nv(vcpu))
334 return 0;
335
336 wi.read_desc = read_guest_s2_desc;
337 wi.data = vcpu;
338 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
339
340 vtcr_to_walk_info(vtcr, &wi);
341
342 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
343
344 ret = walk_nested_s2_pgd(gipa, &wi, result);
345 if (ret)
346 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
347
348 return ret;
349 }
350
ttl_to_size(u8 ttl)351 static unsigned int ttl_to_size(u8 ttl)
352 {
353 int level = ttl & 3;
354 int gran = (ttl >> 2) & 3;
355 unsigned int max_size = 0;
356
357 switch (gran) {
358 case TLBI_TTL_TG_4K:
359 switch (level) {
360 case 0:
361 break;
362 case 1:
363 max_size = SZ_1G;
364 break;
365 case 2:
366 max_size = SZ_2M;
367 break;
368 case 3:
369 max_size = SZ_4K;
370 break;
371 }
372 break;
373 case TLBI_TTL_TG_16K:
374 switch (level) {
375 case 0:
376 case 1:
377 break;
378 case 2:
379 max_size = SZ_32M;
380 break;
381 case 3:
382 max_size = SZ_16K;
383 break;
384 }
385 break;
386 case TLBI_TTL_TG_64K:
387 switch (level) {
388 case 0:
389 case 1:
390 /* No 52bit IPA support */
391 break;
392 case 2:
393 max_size = SZ_512M;
394 break;
395 case 3:
396 max_size = SZ_64K;
397 break;
398 }
399 break;
400 default: /* No size information */
401 break;
402 }
403
404 return max_size;
405 }
406
407 /*
408 * Compute the equivalent of the TTL field by parsing the shadow PT. The
409 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
410 * retrieved from first entry carrying the level as a tag.
411 */
get_guest_mapping_ttl(struct kvm_s2_mmu * mmu,u64 addr)412 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
413 {
414 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
415 kvm_pte_t pte;
416 u8 ttl, level;
417
418 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
419
420 switch (vtcr & VTCR_EL2_TG0_MASK) {
421 case VTCR_EL2_TG0_4K:
422 ttl = (TLBI_TTL_TG_4K << 2);
423 break;
424 case VTCR_EL2_TG0_16K:
425 ttl = (TLBI_TTL_TG_16K << 2);
426 break;
427 case VTCR_EL2_TG0_64K:
428 default: /* IMPDEF: treat any other value as 64k */
429 ttl = (TLBI_TTL_TG_64K << 2);
430 break;
431 }
432
433 tmp = addr;
434
435 again:
436 /* Iteratively compute the block sizes for a particular granule size */
437 switch (vtcr & VTCR_EL2_TG0_MASK) {
438 case VTCR_EL2_TG0_4K:
439 if (sz < SZ_4K) sz = SZ_4K;
440 else if (sz < SZ_2M) sz = SZ_2M;
441 else if (sz < SZ_1G) sz = SZ_1G;
442 else sz = 0;
443 break;
444 case VTCR_EL2_TG0_16K:
445 if (sz < SZ_16K) sz = SZ_16K;
446 else if (sz < SZ_32M) sz = SZ_32M;
447 else sz = 0;
448 break;
449 case VTCR_EL2_TG0_64K:
450 default: /* IMPDEF: treat any other value as 64k */
451 if (sz < SZ_64K) sz = SZ_64K;
452 else if (sz < SZ_512M) sz = SZ_512M;
453 else sz = 0;
454 break;
455 }
456
457 if (sz == 0)
458 return 0;
459
460 tmp &= ~(sz - 1);
461 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
462 goto again;
463 if (!(pte & PTE_VALID))
464 goto again;
465 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
466 if (!level)
467 goto again;
468
469 ttl |= level;
470
471 /*
472 * We now have found some level information in the shadow S2. Check
473 * that the resulting range is actually including the original IPA.
474 */
475 sz = ttl_to_size(ttl);
476 if (addr < (tmp + sz))
477 return ttl;
478
479 return 0;
480 }
481
compute_tlb_inval_range(struct kvm_s2_mmu * mmu,u64 val)482 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
483 {
484 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
485 unsigned long max_size;
486 u8 ttl;
487
488 ttl = FIELD_GET(TLBI_TTL_MASK, val);
489
490 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
491 /* No TTL, check the shadow S2 for a hint */
492 u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
493 ttl = get_guest_mapping_ttl(mmu, addr);
494 }
495
496 max_size = ttl_to_size(ttl);
497
498 if (!max_size) {
499 /* Compute the maximum extent of the invalidation */
500 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) {
501 case VTCR_EL2_TG0_4K:
502 max_size = SZ_1G;
503 break;
504 case VTCR_EL2_TG0_16K:
505 max_size = SZ_32M;
506 break;
507 case VTCR_EL2_TG0_64K:
508 default: /* IMPDEF: treat any other value as 64k */
509 /*
510 * No, we do not support 52bit IPA in nested yet. Once
511 * we do, this should be 4TB.
512 */
513 max_size = SZ_512M;
514 break;
515 }
516 }
517
518 WARN_ON(!max_size);
519 return max_size;
520 }
521
522 /*
523 * We can have multiple *different* MMU contexts with the same VMID:
524 *
525 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit
526 *
527 * - Multiple vcpus using private S2s (huh huh...), hence differing by the
528 * VBBTR_EL2.BADDR address
529 *
530 * - A combination of the above...
531 *
532 * We can always identify which MMU context to pick at run-time. However,
533 * TLB invalidation involving a VMID must take action on all the TLBs using
534 * this particular VMID. This translates into applying the same invalidation
535 * operation to all the contexts that are using this VMID. Moar phun!
536 */
kvm_s2_mmu_iterate_by_vmid(struct kvm * kvm,u16 vmid,const union tlbi_info * info,void (* tlbi_callback)(struct kvm_s2_mmu *,const union tlbi_info *))537 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
538 const union tlbi_info *info,
539 void (*tlbi_callback)(struct kvm_s2_mmu *,
540 const union tlbi_info *))
541 {
542 write_lock(&kvm->mmu_lock);
543
544 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
545 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
546
547 if (!kvm_s2_mmu_valid(mmu))
548 continue;
549
550 if (vmid == get_vmid(mmu->tlb_vttbr))
551 tlbi_callback(mmu, info);
552 }
553
554 write_unlock(&kvm->mmu_lock);
555 }
556
lookup_s2_mmu(struct kvm_vcpu * vcpu)557 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
558 {
559 struct kvm *kvm = vcpu->kvm;
560 bool nested_stage2_enabled;
561 u64 vttbr, vtcr, hcr;
562
563 lockdep_assert_held_write(&kvm->mmu_lock);
564
565 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
566 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
567 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
568
569 nested_stage2_enabled = hcr & HCR_VM;
570
571 /* Don't consider the CnP bit for the vttbr match */
572 vttbr &= ~VTTBR_CNP_BIT;
573
574 /*
575 * Two possibilities when looking up a S2 MMU context:
576 *
577 * - either S2 is enabled in the guest, and we need a context that is
578 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR,
579 * which makes it safe from a TLB conflict perspective (a broken
580 * guest won't be able to generate them),
581 *
582 * - or S2 is disabled, and we need a context that is S2-disabled
583 * and matches the VMID only, as all TLBs are tagged by VMID even
584 * if S2 translation is disabled.
585 */
586 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
587 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
588
589 if (!kvm_s2_mmu_valid(mmu))
590 continue;
591
592 if (nested_stage2_enabled &&
593 mmu->nested_stage2_enabled &&
594 vttbr == mmu->tlb_vttbr &&
595 vtcr == mmu->tlb_vtcr)
596 return mmu;
597
598 if (!nested_stage2_enabled &&
599 !mmu->nested_stage2_enabled &&
600 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr))
601 return mmu;
602 }
603 return NULL;
604 }
605
get_s2_mmu_nested(struct kvm_vcpu * vcpu)606 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
607 {
608 struct kvm *kvm = vcpu->kvm;
609 struct kvm_s2_mmu *s2_mmu;
610 int i;
611
612 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
613
614 s2_mmu = lookup_s2_mmu(vcpu);
615 if (s2_mmu)
616 goto out;
617
618 /*
619 * Make sure we don't always search from the same point, or we
620 * will always reuse a potentially active context, leaving
621 * free contexts unused.
622 */
623 for (i = kvm->arch.nested_mmus_next;
624 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
625 i++) {
626 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
627
628 if (atomic_read(&s2_mmu->refcnt) == 0)
629 break;
630 }
631 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */
632
633 /* Set the scene for the next search */
634 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
635
636 /* Clear the old state */
637 if (kvm_s2_mmu_valid(s2_mmu))
638 kvm_stage2_unmap_range(s2_mmu, 0, kvm_phys_size(s2_mmu));
639
640 /*
641 * The virtual VMID (modulo CnP) will be used as a key when matching
642 * an existing kvm_s2_mmu.
643 *
644 * We cache VTCR at allocation time, once and for all. It'd be great
645 * if the guest didn't screw that one up, as this is not very
646 * forgiving...
647 */
648 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
649 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
650 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
651
652 out:
653 atomic_inc(&s2_mmu->refcnt);
654 return s2_mmu;
655 }
656
kvm_init_nested_s2_mmu(struct kvm_s2_mmu * mmu)657 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
658 {
659 /* CnP being set denotes an invalid entry */
660 mmu->tlb_vttbr = VTTBR_CNP_BIT;
661 mmu->nested_stage2_enabled = false;
662 atomic_set(&mmu->refcnt, 0);
663 }
664
kvm_vcpu_load_hw_mmu(struct kvm_vcpu * vcpu)665 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
666 {
667 if (is_hyp_ctxt(vcpu)) {
668 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
669 } else {
670 write_lock(&vcpu->kvm->mmu_lock);
671 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
672 write_unlock(&vcpu->kvm->mmu_lock);
673 }
674 }
675
kvm_vcpu_put_hw_mmu(struct kvm_vcpu * vcpu)676 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
677 {
678 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) {
679 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
680 vcpu->arch.hw_mmu = NULL;
681 }
682 }
683
684 /*
685 * Returns non-zero if permission fault is handled by injecting it to the next
686 * level hypervisor.
687 */
kvm_s2_handle_perm_fault(struct kvm_vcpu * vcpu,struct kvm_s2_trans * trans)688 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
689 {
690 bool forward_fault = false;
691
692 trans->esr = 0;
693
694 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
695 return 0;
696
697 if (kvm_vcpu_trap_is_iabt(vcpu)) {
698 forward_fault = !kvm_s2_trans_executable(trans);
699 } else {
700 bool write_fault = kvm_is_write_fault(vcpu);
701
702 forward_fault = ((write_fault && !trans->writable) ||
703 (!write_fault && !trans->readable));
704 }
705
706 if (forward_fault)
707 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
708
709 return forward_fault;
710 }
711
kvm_inject_s2_fault(struct kvm_vcpu * vcpu,u64 esr_el2)712 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
713 {
714 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
715 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
716
717 return kvm_inject_nested_sync(vcpu, esr_el2);
718 }
719
kvm_nested_s2_wp(struct kvm * kvm)720 void kvm_nested_s2_wp(struct kvm *kvm)
721 {
722 int i;
723
724 lockdep_assert_held_write(&kvm->mmu_lock);
725
726 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
727 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
728
729 if (kvm_s2_mmu_valid(mmu))
730 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
731 }
732 }
733
kvm_nested_s2_unmap(struct kvm * kvm)734 void kvm_nested_s2_unmap(struct kvm *kvm)
735 {
736 int i;
737
738 lockdep_assert_held_write(&kvm->mmu_lock);
739
740 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
741 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
742
743 if (kvm_s2_mmu_valid(mmu))
744 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu));
745 }
746 }
747
kvm_nested_s2_flush(struct kvm * kvm)748 void kvm_nested_s2_flush(struct kvm *kvm)
749 {
750 int i;
751
752 lockdep_assert_held_write(&kvm->mmu_lock);
753
754 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
755 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
756
757 if (kvm_s2_mmu_valid(mmu))
758 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
759 }
760 }
761
kvm_arch_flush_shadow_all(struct kvm * kvm)762 void kvm_arch_flush_shadow_all(struct kvm *kvm)
763 {
764 int i;
765
766 for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
767 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
768
769 if (!WARN_ON(atomic_read(&mmu->refcnt)))
770 kvm_free_stage2_pgd(mmu);
771 }
772 kvfree(kvm->arch.nested_mmus);
773 kvm->arch.nested_mmus = NULL;
774 kvm->arch.nested_mmus_size = 0;
775 kvm_uninit_stage2_mmu(kvm);
776 }
777
778 /*
779 * Our emulated CPU doesn't support all the possible features. For the
780 * sake of simplicity (and probably mental sanity), wipe out a number
781 * of feature bits we don't intend to support for the time being.
782 * This list should get updated as new features get added to the NV
783 * support, and new extension to the architecture.
784 */
limit_nv_id_regs(struct kvm * kvm)785 static void limit_nv_id_regs(struct kvm *kvm)
786 {
787 u64 val, tmp;
788
789 /* Support everything but TME */
790 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1);
791 val &= ~NV_FTR(ISAR0, TME);
792 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1, val);
793
794 /* Support everything but Spec Invalidation and LS64 */
795 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1);
796 val &= ~(NV_FTR(ISAR1, LS64) |
797 NV_FTR(ISAR1, SPECRES));
798 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val);
799
800 /* No AMU, MPAM, S-EL2, or RAS */
801 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1);
802 val &= ~(GENMASK_ULL(55, 52) |
803 NV_FTR(PFR0, AMU) |
804 NV_FTR(PFR0, MPAM) |
805 NV_FTR(PFR0, SEL2) |
806 NV_FTR(PFR0, RAS) |
807 NV_FTR(PFR0, EL3) |
808 NV_FTR(PFR0, EL2) |
809 NV_FTR(PFR0, EL1));
810 /* 64bit EL1/EL2/EL3 only */
811 val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001);
812 val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001);
813 val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001);
814 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val);
815
816 /* Only support BTI, SSBS, CSV2_frac */
817 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1);
818 val &= (NV_FTR(PFR1, BT) |
819 NV_FTR(PFR1, SSBS) |
820 NV_FTR(PFR1, CSV2_frac));
821 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1, val);
822
823 /* Hide ECV, ExS, Secure Memory */
824 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
825 val &= ~(NV_FTR(MMFR0, ECV) |
826 NV_FTR(MMFR0, EXS) |
827 NV_FTR(MMFR0, TGRAN4_2) |
828 NV_FTR(MMFR0, TGRAN16_2) |
829 NV_FTR(MMFR0, TGRAN64_2) |
830 NV_FTR(MMFR0, SNSMEM));
831
832 /* Disallow unsupported S2 page sizes */
833 switch (PAGE_SIZE) {
834 case SZ_64K:
835 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001);
836 fallthrough;
837 case SZ_16K:
838 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001);
839 fallthrough;
840 case SZ_4K:
841 /* Support everything */
842 break;
843 }
844 /*
845 * Since we can't support a guest S2 page size smaller than
846 * the host's own page size (due to KVM only populating its
847 * own S2 using the kernel's page size), advertise the
848 * limitation using FEAT_GTG.
849 */
850 switch (PAGE_SIZE) {
851 case SZ_4K:
852 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010);
853 fallthrough;
854 case SZ_16K:
855 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010);
856 fallthrough;
857 case SZ_64K:
858 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010);
859 break;
860 }
861 /* Cap PARange to 48bits */
862 tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val);
863 if (tmp > 0b0101) {
864 val &= ~NV_FTR(MMFR0, PARANGE);
865 val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101);
866 }
867 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1, val);
868
869 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1);
870 val &= (NV_FTR(MMFR1, HCX) |
871 NV_FTR(MMFR1, PAN) |
872 NV_FTR(MMFR1, LO) |
873 NV_FTR(MMFR1, HPDS) |
874 NV_FTR(MMFR1, VH) |
875 NV_FTR(MMFR1, VMIDBits));
876 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1, val);
877
878 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1);
879 val &= ~(NV_FTR(MMFR2, BBM) |
880 NV_FTR(MMFR2, TTL) |
881 GENMASK_ULL(47, 44) |
882 NV_FTR(MMFR2, ST) |
883 NV_FTR(MMFR2, CCIDX) |
884 NV_FTR(MMFR2, VARange));
885
886 /* Force TTL support */
887 val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001);
888 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1, val);
889
890 val = 0;
891 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1))
892 val |= FIELD_PREP(NV_FTR(MMFR4, E2H0),
893 ID_AA64MMFR4_EL1_E2H0_NI_NV1);
894 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR4_EL1, val);
895
896 /* Only limited support for PMU, Debug, BPs and WPs */
897 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1);
898 val &= (NV_FTR(DFR0, PMUVer) |
899 NV_FTR(DFR0, WRPs) |
900 NV_FTR(DFR0, BRPs) |
901 NV_FTR(DFR0, DebugVer));
902
903 /* Cap Debug to ARMv8.1 */
904 tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val);
905 if (tmp > 0b0111) {
906 val &= ~NV_FTR(DFR0, DebugVer);
907 val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111);
908 }
909 kvm_set_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1, val);
910 }
911
kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu * vcpu,enum vcpu_sysreg sr)912 u64 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg sr)
913 {
914 u64 v = ctxt_sys_reg(&vcpu->arch.ctxt, sr);
915 struct kvm_sysreg_masks *masks;
916
917 masks = vcpu->kvm->arch.sysreg_masks;
918
919 if (masks) {
920 sr -= __VNCR_START__;
921
922 v &= ~masks->mask[sr].res0;
923 v |= masks->mask[sr].res1;
924 }
925
926 return v;
927 }
928
set_sysreg_masks(struct kvm * kvm,int sr,u64 res0,u64 res1)929 static void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1)
930 {
931 int i = sr - __VNCR_START__;
932
933 kvm->arch.sysreg_masks->mask[i].res0 = res0;
934 kvm->arch.sysreg_masks->mask[i].res1 = res1;
935 }
936
kvm_init_nv_sysregs(struct kvm * kvm)937 int kvm_init_nv_sysregs(struct kvm *kvm)
938 {
939 u64 res0, res1;
940
941 lockdep_assert_held(&kvm->arch.config_lock);
942
943 if (kvm->arch.sysreg_masks)
944 return 0;
945
946 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)),
947 GFP_KERNEL_ACCOUNT);
948 if (!kvm->arch.sysreg_masks)
949 return -ENOMEM;
950
951 limit_nv_id_regs(kvm);
952
953 /* VTTBR_EL2 */
954 res0 = res1 = 0;
955 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
956 res0 |= GENMASK(63, 56);
957 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
958 res0 |= VTTBR_CNP_BIT;
959 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1);
960
961 /* VTCR_EL2 */
962 res0 = GENMASK(63, 32) | GENMASK(30, 20);
963 res1 = BIT(31);
964 set_sysreg_masks(kvm, VTCR_EL2, res0, res1);
965
966 /* VMPIDR_EL2 */
967 res0 = GENMASK(63, 40) | GENMASK(30, 24);
968 res1 = BIT(31);
969 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1);
970
971 /* HCR_EL2 */
972 res0 = BIT(48);
973 res1 = HCR_RW;
974 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP))
975 res0 |= GENMASK(63, 59);
976 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2))
977 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA);
978 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS))
979 res0 |= (HCR_TTLBIS | HCR_TTLBOS);
980 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
981 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
982 res0 |= HCR_ENSCXT;
983 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP))
984 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4);
985 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
986 res0 |= HCR_AMVOFFEN;
987 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1))
988 res0 |= HCR_FIEN;
989 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP))
990 res0 |= HCR_FWB;
991 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2))
992 res0 |= HCR_NV2;
993 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP))
994 res0 |= (HCR_AT | HCR_NV1 | HCR_NV);
995 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
996 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
997 res0 |= (HCR_API | HCR_APK);
998 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP))
999 res0 |= BIT(39);
1000 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1001 res0 |= (HCR_TEA | HCR_TERR);
1002 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1003 res0 |= HCR_TLOR;
1004 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP))
1005 res1 |= HCR_E2H;
1006 set_sysreg_masks(kvm, HCR_EL2, res0, res1);
1007
1008 /* HCRX_EL2 */
1009 res0 = HCRX_EL2_RES0;
1010 res1 = HCRX_EL2_RES1;
1011 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP))
1012 res0 |= HCRX_EL2_PACMEn;
1013 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP))
1014 res0 |= HCRX_EL2_EnFPM;
1015 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1016 res0 |= HCRX_EL2_GCSEn;
1017 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP))
1018 res0 |= HCRX_EL2_EnIDCP128;
1019 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC))
1020 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR);
1021 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP))
1022 res0 |= HCRX_EL2_TMEA;
1023 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
1024 res0 |= HCRX_EL2_D128En;
1025 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1026 res0 |= HCRX_EL2_PTTWI;
1027 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP))
1028 res0 |= HCRX_EL2_SCTLR2En;
1029 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, TCRX, IMP))
1030 res0 |= HCRX_EL2_TCR2En;
1031 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
1032 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
1033 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP))
1034 res0 |= HCRX_EL2_CMOW;
1035 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP))
1036 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT);
1037 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) ||
1038 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS))
1039 res0 |= HCRX_EL2_SMPME;
1040 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))
1041 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS);
1042 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V))
1043 res0 |= HCRX_EL2_EnASR;
1044 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64))
1045 res0 |= HCRX_EL2_EnALS;
1046 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1047 res0 |= HCRX_EL2_EnAS0;
1048 set_sysreg_masks(kvm, HCRX_EL2, res0, res1);
1049
1050 /* HFG[RW]TR_EL2 */
1051 res0 = res1 = 0;
1052 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) &&
1053 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC)))
1054 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey |
1055 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey |
1056 HFGxTR_EL2_APIBKey);
1057 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
1058 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 |
1059 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 |
1060 HFGxTR_EL2_LORSA_EL1);
1061 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) &&
1062 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2))
1063 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0);
1064 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP))
1065 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1;
1066 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP))
1067 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 |
1068 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 |
1069 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 |
1070 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 |
1071 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1);
1072 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA))
1073 res0 |= HFGxTR_EL2_nACCDATA_EL1;
1074 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1075 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1);
1076 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP))
1077 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0);
1078 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
1079 res0 |= HFGxTR_EL2_nRCWMASK_EL1;
1080 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1PIE, IMP))
1081 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1);
1082 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1POE, IMP))
1083 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1);
1084 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP))
1085 res0 |= HFGxTR_EL2_nS2POR_EL1;
1086 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
1087 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1);
1088 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1);
1089 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1);
1090
1091 /* HDFG[RW]TR_EL2 */
1092 res0 = res1 = 0;
1093 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP))
1094 res0 |= HDFGRTR_EL2_OSDLR_EL1;
1095 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1096 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 |
1097 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 |
1098 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN |
1099 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 |
1100 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 |
1101 HDFGRTR_EL2_PMCEIDn_EL0);
1102 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
1103 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 |
1104 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 |
1105 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 |
1106 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 |
1107 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 |
1108 HDFGRTR_EL2_PMBIDR_EL1);
1109 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1110 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS |
1111 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM |
1112 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID |
1113 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR |
1114 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR |
1115 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR |
1116 HDFGRTR_EL2_TRCVICTLR);
1117 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
1118 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 |
1119 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 |
1120 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 |
1121 HDFGRTR_EL2_TRBTRG_EL1);
1122 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1123 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL |
1124 HDFGRTR_EL2_nBRBDATA);
1125 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
1126 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1;
1127 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1);
1128
1129 /* Reuse the bits from the read-side and add the write-specific stuff */
1130 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
1131 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0);
1132 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP))
1133 res0 |= HDFGWTR_EL2_TRCOSLAR;
1134 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
1135 res0 |= HDFGWTR_EL2_TRFCR_EL1;
1136 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1);
1137
1138 /* HFGITR_EL2 */
1139 res0 = HFGITR_EL2_RES0;
1140 res1 = HFGITR_EL2_RES1;
1141 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2))
1142 res0 |= HFGITR_EL2_DCCVADP;
1143 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2))
1144 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP);
1145 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
1146 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1147 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS |
1148 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS |
1149 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS |
1150 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS);
1151 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
1152 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 |
1153 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 |
1154 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS |
1155 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS |
1156 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS |
1157 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS);
1158 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP))
1159 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX |
1160 HFGITR_EL2_CPPRCTX);
1161 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP))
1162 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL);
1163 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP))
1164 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 |
1165 HFGITR_EL2_nGCSEPP);
1166 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX))
1167 res0 |= HFGITR_EL2_COSPRCTX;
1168 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP))
1169 res0 |= HFGITR_EL2_ATS1E1A;
1170 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1);
1171
1172 /* HAFGRTR_EL2 - not a lot to see here */
1173 res0 = HAFGRTR_EL2_RES0;
1174 res1 = HAFGRTR_EL2_RES1;
1175 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1))
1176 res0 |= ~(res0 | res1);
1177 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1);
1178
1179 /* SCTLR_EL1 */
1180 res0 = SCTLR_EL1_RES0;
1181 res1 = SCTLR_EL1_RES1;
1182 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
1183 res0 |= SCTLR_EL1_EPAN;
1184 set_sysreg_masks(kvm, SCTLR_EL1, res0, res1);
1185
1186 return 0;
1187 }
1188