Lines Matching +full:force +full:- +full:hpd
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2017 - Linaro Ltd
15 wr->fst = fst; in fail_s1_walk()
16 wr->ptw = s1ptw; in fail_s1_walk()
17 wr->s2 = s1ptw; in fail_s1_walk()
18 wr->failed = true; in fail_s1_walk()
21 #define S1_MMU_DISABLED (-127)
25 return 64 - wi->txsz; in get_ia_size()
31 if (wi->pa52bit) in check_output_size()
32 return wi->max_oa_bits < 52 && (ipa & GENMASK_ULL(51, wi->max_oa_bits)); in check_output_size()
33 return wi->max_oa_bits < 48 && (ipa & GENMASK_ULL(47, wi->max_oa_bits)); in check_output_size()
38 switch (BIT(wi->pgshift)) { in has_52bit_pa()
41 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52)) in has_52bit_pa()
43 return ((wi->regime == TR_EL2 ? in has_52bit_pa()
47 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)) in has_52bit_pa()
51 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)) in has_52bit_pa()
56 return (tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS)); in has_52bit_pa()
63 if (!wi->pa52bit) in desc_to_oa()
64 return desc & GENMASK_ULL(47, wi->pgshift); in desc_to_oa()
66 switch (BIT(wi->pgshift)) { in desc_to_oa()
69 addr = desc & GENMASK_ULL(49, wi->pgshift); in desc_to_oa()
74 addr = desc & GENMASK_ULL(47, wi->pgshift); in desc_to_oa()
115 if (!kvm_has_s1pie(vcpu->kvm)) in s1pie_enabled()
126 if (!kvm_has_s1poe(vcpu->kvm)) { in compute_s1poe()
127 wi->poe = wi->e0poe = false; in compute_s1poe()
131 val = effective_tcr2(vcpu, wi->regime); in compute_s1poe()
134 wi->poe = val & TCR2_EL1_POE; in compute_s1poe()
135 wi->e0poe = (wi->regime != TR_EL2) && (val & TCR2_EL1_E0POE); in compute_s1poe()
149 wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); in setup_s1_walk()
151 WARN_ON_ONCE(wi->regime != TR_EL10); in setup_s1_walk()
152 wi->s2 = false; in setup_s1_walk()
156 switch (wi->regime) { in setup_s1_walk()
177 if (va55 && wi->regime != TR_EL2) { in setup_s1_walk()
178 wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr); in setup_s1_walk()
183 wi->pgshift = 12; break; in setup_s1_walk()
185 wi->pgshift = 14; break; in setup_s1_walk()
188 wi->pgshift = 16; break; in setup_s1_walk()
191 wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr); in setup_s1_walk()
196 wi->pgshift = 12; break; in setup_s1_walk()
198 wi->pgshift = 14; break; in setup_s1_walk()
201 wi->pgshift = 16; break; in setup_s1_walk()
205 wi->pa52bit = has_52bit_pa(vcpu, wi, tcr); in setup_s1_walk()
210 stride = wi->pgshift - 3; in setup_s1_walk()
211 wi->sl = 3 - (((ia_bits - 1) - wi->pgshift) / stride); in setup_s1_walk()
213 if (wi->regime == TR_EL2 && va55) in setup_s1_walk()
216 tbi = (wi->regime == TR_EL2 ? in setup_s1_walk()
225 wi->sh = (wi->regime == TR_EL2 ? in setup_s1_walk()
234 switch (wi->regime) { in setup_s1_walk()
240 * - HCR_EL2.DC = 1 in setup_s1_walk()
241 * - HCR_EL2.{E2H,TGE} = {0,1} in setup_s1_walk()
242 * - SCTLR_EL1.M = 0 in setup_s1_walk()
249 wr->level = S1_MMU_DISABLED; in setup_s1_walk()
256 wr->level = S1_MMU_DISABLED; in setup_s1_walk()
260 if (wr->level == S1_MMU_DISABLED) { in setup_s1_walk()
261 if (va >= BIT(kvm_get_pa_bits(vcpu->kvm))) in setup_s1_walk()
264 wr->pa = va; in setup_s1_walk()
268 wi->be = sctlr & SCTLR_ELx_EE; in setup_s1_walk()
270 wi->hpd = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HPDS, IMP); in setup_s1_walk()
271 wi->hpd &= (wi->regime == TR_EL2 ? in setup_s1_walk()
277 wi->hpd |= s1pie_enabled(vcpu, wi->regime); in setup_s1_walk()
283 wi->hpd |= (wi->poe || wi->e0poe); in setup_s1_walk()
286 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR2_EL1, ST, 48_47)) { in setup_s1_walk()
287 if (wi->txsz > 39) in setup_s1_walk()
290 if (wi->txsz > 48 || (BIT(wi->pgshift) == SZ_64K && wi->txsz > 47)) in setup_s1_walk()
295 switch (BIT(wi->pgshift)) { in setup_s1_walk()
298 lva = wi->pa52bit; in setup_s1_walk()
301 lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, VARange, 52); in setup_s1_walk()
305 if ((lva && wi->txsz < 12) || (!lva && wi->txsz < 16)) in setup_s1_walk()
309 if ((!va55 && va > GENMASK(ia_bits - 1, 0)) || in setup_s1_walk()
314 if (wi->regime != TR_EL2 && in setup_s1_walk()
319 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && in setup_s1_walk()
320 wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) in setup_s1_walk()
323 ps = (wi->regime == TR_EL2 ? in setup_s1_walk()
326 wi->max_oa_bits = min(get_kvm_ipa_limit(), ps_to_output_size(ps, wi->pa52bit)); in setup_s1_walk()
329 x = 3 + ia_bits - ((3 - wi->sl) * stride + wi->pgshift); in setup_s1_walk()
331 wi->baddr = ttbr & TTBRx_EL1_BADDR; in setup_s1_walk()
332 if (wi->pa52bit) { in setup_s1_walk()
334 * Force the alignment on 64 bytes for top-level tables in setup_s1_walk()
340 wi->baddr |= FIELD_GET(GENMASK_ULL(5, 2), ttbr) << 48; in setup_s1_walk()
344 if (check_output_size(wi->baddr, wi)) in setup_s1_walk()
347 wi->baddr &= GENMASK_ULL(wi->max_oa_bits - 1, x); in setup_s1_walk()
357 return -EFAULT; in setup_s1_walk()
361 fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(wi->sl), false); in setup_s1_walk()
362 return -EFAULT; in setup_s1_walk()
371 level = wi->sl; in walk_s1()
372 stride = wi->pgshift - 3; in walk_s1()
373 baddr = wi->baddr; in walk_s1()
375 va_top = get_ia_size(wi) - 1; in walk_s1()
380 va_bottom = (3 - level) * stride + wi->pgshift; in walk_s1()
381 index = (va & GENMASK_ULL(va_top, va_bottom)) >> (va_bottom - 3); in walk_s1()
385 if (wi->s2) { in walk_s1()
400 return -EPERM; in walk_s1()
406 if (wi->filter) { in walk_s1()
407 ret = wi->filter->fn(&(struct s1_walk_context) in walk_s1()
412 }, wi->filter->priv); in walk_s1()
417 ret = kvm_read_guest(vcpu->kvm, ipa, &desc, sizeof(desc)); in walk_s1()
423 if (wi->be) in walk_s1()
441 if (!wi->hpd) { in walk_s1()
442 wr->APTable |= FIELD_GET(S1_TABLE_AP, desc); in walk_s1()
443 wr->UXNTable |= FIELD_GET(PMD_TABLE_UXN, desc); in walk_s1()
444 wr->PXNTable |= FIELD_GET(PMD_TABLE_PXN, desc); in walk_s1()
449 /* Check for out-of-range OA */ in walk_s1()
454 va_top = va_bottom - 1; in walk_s1()
462 switch (BIT(wi->pgshift)) { in walk_s1()
464 valid_block = level == 1 || level == 2 || (wi->pa52bit && level == 0); in walk_s1()
468 valid_block = level == 2 || (wi->pa52bit && level == 1); in walk_s1()
482 return -EACCES; in walk_s1()
487 wr->failed = false; in walk_s1()
488 wr->level = level; in walk_s1()
489 wr->desc = desc; in walk_s1()
490 wr->pa = baddr & GENMASK(52, va_bottom); in walk_s1()
491 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); in walk_s1()
493 wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); in walk_s1()
494 if (wr->nG) { in walk_s1()
497 switch (wi->regime) { in walk_s1()
514 wr->asid = FIELD_GET(TTBR_ASID_MASK, asid_ttbr); in walk_s1()
515 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || in walk_s1()
517 wr->asid &= GENMASK(7, 0); in walk_s1()
524 return -EINVAL; in walk_s1()
527 return -ENOENT; in walk_s1()
547 config->ttbr0 = read_sysreg_el1(SYS_TTBR0); in __mmu_config_save()
548 config->ttbr1 = read_sysreg_el1(SYS_TTBR1); in __mmu_config_save()
549 config->tcr = read_sysreg_el1(SYS_TCR); in __mmu_config_save()
550 config->mair = read_sysreg_el1(SYS_MAIR); in __mmu_config_save()
552 config->tcr2 = read_sysreg_el1(SYS_TCR2); in __mmu_config_save()
554 config->pir = read_sysreg_el1(SYS_PIR); in __mmu_config_save()
555 config->pire0 = read_sysreg_el1(SYS_PIRE0); in __mmu_config_save()
558 config->por_el1 = read_sysreg_el1(SYS_POR); in __mmu_config_save()
559 config->por_el0 = read_sysreg_s(SYS_POR_EL0); in __mmu_config_save()
562 config->sctlr = read_sysreg_el1(SYS_SCTLR); in __mmu_config_save()
563 config->vttbr = read_sysreg(vttbr_el2); in __mmu_config_save()
564 config->vtcr = read_sysreg(vtcr_el2); in __mmu_config_save()
575 write_sysreg_el1(config->ttbr0, SYS_TTBR0); in __mmu_config_restore()
576 write_sysreg_el1(config->ttbr1, SYS_TTBR1); in __mmu_config_restore()
577 write_sysreg_el1(config->tcr, SYS_TCR); in __mmu_config_restore()
578 write_sysreg_el1(config->mair, SYS_MAIR); in __mmu_config_restore()
580 write_sysreg_el1(config->tcr2, SYS_TCR2); in __mmu_config_restore()
582 write_sysreg_el1(config->pir, SYS_PIR); in __mmu_config_restore()
583 write_sysreg_el1(config->pire0, SYS_PIRE0); in __mmu_config_restore()
586 write_sysreg_el1(config->por_el1, SYS_POR); in __mmu_config_restore()
587 write_sysreg_s(config->por_el0, SYS_POR_EL0); in __mmu_config_restore()
590 write_sysreg_el1(config->sctlr, SYS_SCTLR); in __mmu_config_restore()
591 write_sysreg(config->vttbr, vttbr_el2); in __mmu_config_restore()
592 write_sysreg(config->vtcr, vtcr_el2); in __mmu_config_restore()
671 /* Upgrade transient s1 to non-transient to simplify things */ in combine_s1_s2_attr()
673 case 0b0001 ... 0b0011: /* Normal, Write-Through Transient */ in combine_s1_s2_attr()
677 case 0b0101 ... 0b0111: /* Normal, Write-Back Transient */ in combine_s1_s2_attr()
737 * non-52bit and LPA have their basic shareability described in the in compute_s1_sh()
741 if (!wi->pa52bit || BIT(wi->pgshift) == SZ_64K) in compute_s1_sh()
742 sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_SH, wr->desc); in compute_s1_sh()
744 sh = wi->sh; in compute_s1_sh()
766 if (tr->esr) { in compute_par_s12()
770 par |= FIELD_PREP(SYS_PAR_EL1_FST, tr->esr); in compute_par_s12()
775 s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc); in compute_par_s12()
778 if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP)) in compute_par_s12()
801 /* Reserved, do something non-silly */ in compute_par_s12()
835 s2_sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S2_SH, tr->desc); in compute_par_s12()
838 par |= tr->output & GENMASK(47, 12); in compute_par_s12()
851 if (wr->failed) { in compute_par_s1()
854 par |= FIELD_PREP(SYS_PAR_EL1_FST, wr->fst); in compute_par_s1()
855 par |= wr->ptw ? SYS_PAR_EL1_PTW : 0; in compute_par_s1()
856 par |= wr->s2 ? SYS_PAR_EL1_S : 0; in compute_par_s1()
857 } else if (wr->level == S1_MMU_DISABLED) { in compute_par_s1()
860 par |= wr->pa & SYS_PAR_EL1_PA; in compute_par_s1()
862 if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && in compute_par_s1()
877 mair = (wi->regime == TR_EL10 ? in compute_par_s1()
881 mair >>= FIELD_GET(PTE_ATTRINDX_MASK, wr->desc) * 8; in compute_par_s1()
884 sctlr = (wi->regime == TR_EL10 ? in compute_par_s1()
888 /* Force NC for memory if SCTLR_ELx.C is clear */ in compute_par_s1()
893 par |= wr->pa & SYS_PAR_EL1_PA; in compute_par_s1()
906 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) in pan3_enabled()
926 /* Non-hierarchical part of AArch64.S1DirectBasePermissions() */ in compute_s1_direct_permissions()
927 if (wi->regime != TR_EL2) { in compute_s1_direct_permissions()
928 switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr->desc)) { in compute_s1_direct_permissions()
930 wr->pr = wr->pw = true; in compute_s1_direct_permissions()
931 wr->ur = wr->uw = false; in compute_s1_direct_permissions()
934 wr->pr = wr->pw = wr->ur = wr->uw = true; in compute_s1_direct_permissions()
937 wr->pr = true; in compute_s1_direct_permissions()
938 wr->pw = wr->ur = wr->uw = false; in compute_s1_direct_permissions()
941 wr->pr = wr->ur = true; in compute_s1_direct_permissions()
942 wr->pw = wr->uw = false; in compute_s1_direct_permissions()
947 wr->px = !((wr->desc & PTE_PXN) || wr->uw); in compute_s1_direct_permissions()
948 wr->ux = !(wr->desc & PTE_UXN); in compute_s1_direct_permissions()
950 wr->ur = wr->uw = wr->ux = false; in compute_s1_direct_permissions()
952 if (!(wr->desc & PTE_RDONLY)) { in compute_s1_direct_permissions()
953 wr->pr = wr->pw = true; in compute_s1_direct_permissions()
955 wr->pr = true; in compute_s1_direct_permissions()
956 wr->pw = false; in compute_s1_direct_permissions()
960 wr->px = !(wr->desc & PTE_UXN); in compute_s1_direct_permissions()
963 switch (wi->regime) { in compute_s1_direct_permissions()
973 wr->pwxn = wr->uwxn = wxn; in compute_s1_direct_permissions()
974 wr->pov = wi->poe; in compute_s1_direct_permissions()
975 wr->uov = wi->e0poe; in compute_s1_direct_permissions()
983 if (wi->regime != TR_EL2) { in compute_s1_hierarchical_permissions()
984 switch (wr->APTable) { in compute_s1_hierarchical_permissions()
988 wr->ur = wr->uw = false; in compute_s1_hierarchical_permissions()
991 wr->pw = wr->uw = false; in compute_s1_hierarchical_permissions()
994 wr->pw = wr->ur = wr->uw = false; in compute_s1_hierarchical_permissions()
998 wr->px &= !wr->PXNTable; in compute_s1_hierarchical_permissions()
999 wr->ux &= !wr->UXNTable; in compute_s1_hierarchical_permissions()
1001 if (wr->APTable & BIT(1)) in compute_s1_hierarchical_permissions()
1002 wr->pw = false; in compute_s1_hierarchical_permissions()
1005 wr->px &= !wr->UXNTable; in compute_s1_hierarchical_permissions()
1013 (wr)->pr = (r); \
1014 (wr)->pw = (w); \
1015 (wr)->px = (x); \
1020 (wr)->ur = (r); \
1021 (wr)->uw = (w); \
1022 (wr)->ux = (x); \
1027 (wr)->pwxn = (v); \
1032 (wr)->uwxn = (v); \
1101 idx = pte_pi_index(wr->desc); in compute_s1_indirect_permissions()
1103 switch (wi->regime) { in compute_s1_indirect_permissions()
1120 if (wi->regime != TR_EL2) in compute_s1_indirect_permissions()
1125 wr->pov = wi->poe && !(pp & BIT(3)); in compute_s1_indirect_permissions()
1126 wr->uov = wi->e0poe && !(up & BIT(3)); in compute_s1_indirect_permissions()
1129 if (wr->px && wr->uw) { in compute_s1_indirect_permissions()
1141 idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc); in compute_s1_overlay_permissions()
1143 if (wr->pov) { in compute_s1_overlay_permissions()
1144 switch (wi->regime) { in compute_s1_overlay_permissions()
1160 if (wr->pwxn && (pov_perms & POE_X)) in compute_s1_overlay_permissions()
1163 wr->pr &= pov_perms & POE_R; in compute_s1_overlay_permissions()
1164 wr->pw &= pov_perms & POE_W; in compute_s1_overlay_permissions()
1165 wr->px &= pov_perms & POE_X; in compute_s1_overlay_permissions()
1168 if (wr->uov) { in compute_s1_overlay_permissions()
1169 switch (wi->regime) { in compute_s1_overlay_permissions()
1185 if (wr->uwxn && (uov_perms & POE_X)) in compute_s1_overlay_permissions()
1188 wr->ur &= uov_perms & POE_R; in compute_s1_overlay_permissions()
1189 wr->uw &= uov_perms & POE_W; in compute_s1_overlay_permissions()
1190 wr->ux &= uov_perms & POE_X; in compute_s1_overlay_permissions()
1200 if (!s1pie_enabled(vcpu, wi->regime)) in compute_s1_permissions()
1205 if (!wi->hpd) in compute_s1_permissions()
1211 if (!wr->pov) in compute_s1_permissions()
1212 wr->px &= !(wr->pwxn && wr->pw); in compute_s1_permissions()
1215 if (!wr->uov) in compute_s1_permissions()
1216 wr->ux &= !(wr->uwxn && wr->uw); in compute_s1_permissions()
1218 pan = wi->pan && (wr->ur || wr->uw || in compute_s1_permissions()
1219 (pan3_enabled(vcpu, wi->regime) && wr->ux)); in compute_s1_permissions()
1220 wr->pw &= !pan; in compute_s1_permissions()
1221 wr->pr &= !pan; in compute_s1_permissions()
1243 idx = srcu_read_lock(&vcpu->kvm->srcu); in handle_at_slow()
1247 srcu_read_unlock(&vcpu->kvm->srcu, idx); in handle_at_slow()
1308 guard(write_lock_irqsave)(&vcpu->kvm->mmu_lock); in __kvm_at_s1e01_fast()
1338 if (kvm_has_tcr2(vcpu->kvm)) { in __kvm_at_s1e01_fast()
1340 if (kvm_has_s1pie(vcpu->kvm)) { in __kvm_at_s1e01_fast()
1344 if (kvm_has_s1poe(vcpu->kvm)) { in __kvm_at_s1e01_fast()
1350 __load_stage2(mmu, mmu->arch); in __kvm_at_s1e01_fast()
1354 write_sysreg_hcr(vcpu->arch.hcr_el2); in __kvm_at_s1e01_fast()
1438 scoped_guard(write_lock_irqsave, &vcpu->kvm->mmu_lock) { in __kvm_at_s1e2()
1492 /* Do the stage-1 translation */ in __kvm_at_s12()
1528 /* Do the stage-2 translation */ in __kvm_at_s12()
1546 * or without PAN. This requires wi->{regime, as_el0, pan} to be
1547 * set. The rest of the wi and wr should be 0-initialised.
1558 if (wr->level == S1_MMU_DISABLED) { in __kvm_translate_va()
1559 wr->ur = wr->uw = wr->ux = true; in __kvm_translate_va()
1560 wr->pr = wr->pw = wr->px = true; in __kvm_translate_va()
1580 u64 ipa = dm->ipa; in match_s1_desc()
1583 ipa &= GENMASK(51, ctxt->wi->pgshift); in match_s1_desc()
1586 if (ipa != ctxt->table_ipa) in match_s1_desc()
1590 dm->level = ctxt->level; in match_s1_desc()
1591 return -EINTR; in match_s1_desc()
1628 case -EINTR: in __kvm_find_s1_desc_level()
1634 return -ENOENT; in __kvm_find_s1_desc_level()