Lines Matching defs:vcpu
41 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
83 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
85 struct kvm *kvm = vcpu->kvm;
94 if (!vcpu->arch.ctxt.vncr_array)
95 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
98 if (!vcpu->arch.ctxt.vncr_array)
122 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
123 vcpu->arch.ctxt.vncr_array = NULL;
153 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc)
157 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC;
167 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi,
172 pa_max = kvm_get_pa_bits(vcpu->kvm);
213 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc,
219 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
235 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,
246 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
256 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
284 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride);
310 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);
366 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);
478 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi)
480 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
482 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
484 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);
490 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
493 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
501 if (!vcpu_has_nv(vcpu))
504 setup_s2_walk(vcpu, &wi);
506 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
508 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
760 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
762 struct kvm *kvm = vcpu->kvm;
768 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
769 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
770 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
809 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
811 struct kvm *kvm = vcpu->kvm;
815 lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
817 s2_mmu = lookup_s2_mmu(vcpu);
853 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
854 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
855 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
869 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu);
882 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
888 if (is_hyp_ctxt(vcpu)) {
889 if (!vcpu->arch.hw_mmu)
890 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
892 if (!vcpu->arch.hw_mmu) {
893 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
894 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
897 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
898 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
917 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu)
922 BUG_ON(is_hyp_ctxt(vcpu));
927 * another vcpu with VNCR still mapped if the TLBI thread is
931 * the back of an running vcpu. This will only generate a fault and
934 if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1)
939 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
942 this_cpu_reset_vncr_fixmap(vcpu);
949 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI))
952 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
953 atomic_dec(&vcpu->arch.hw_mmu->refcnt);
955 vcpu->arch.hw_mmu = NULL;
962 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans)
968 if (!kvm_vcpu_trap_is_permission_fault(vcpu))
971 if (kvm_vcpu_trap_is_iabt(vcpu)) {
972 if (vcpu_mode_priv(vcpu))
973 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
975 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
977 bool write_fault = kvm_is_write_fault(vcpu);
984 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM);
989 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2)
991 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2);
992 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2);
994 return kvm_inject_nested_sync(vcpu, esr_el2);
997 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime)
1005 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
1009 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1016 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx));
1017 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1048 * either can race against a vcpu not being onlined yet (no pseudo-TLB
1059 struct kvm_vcpu *vcpu;
1074 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm)
1095 struct kvm_vcpu *vcpu;
1110 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
1139 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
1256 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1260 compute_s1_tlbi_range(vcpu, inst, val, &scope);
1262 guard(write_lock)(&vcpu->kvm->mmu_lock);
1263 invalidate_vncr_va(vcpu->kvm, &scope);
1346 * - We introduce an internal representation of a vcpu-private TLB,
1373 * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1391 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1393 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1396 if (!vcpu->arch.vncr_tlb) {
1397 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb,
1406 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
1407 vcpu->arch.vncr_tlb = vt;
1410 if (!vcpu->arch.vncr_tlb)
1416 static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1418 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1421 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
1431 vt = vcpu->arch.vncr_tlb;
1435 * current TLB, as it could be sampled from another vcpu doing a
1441 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1442 this_cpu_reset_vncr_fixmap(vcpu);
1444 invalidate_vncr(vcpu->kvm, vt);
1454 guard(srcu)(&vcpu->kvm->srcu);
1456 va = read_vncr_el2(vcpu);
1458 mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1461 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1465 write_fault = kvm_is_write_fault(vcpu);
1468 memslot = gfn_to_memslot(vcpu->kvm, gfn);
1483 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
1485 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
1499 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1504 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1505 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
1506 kvm_release_faultin_page(vcpu->kvm, page, true, false);
1516 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1517 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
1521 mark_page_dirty(vcpu->kvm, gfn);
1526 static void handle_vncr_perm(struct kvm_vcpu *vcpu)
1528 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1529 u64 esr = kvm_vcpu_get_esr(vcpu);
1536 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable)
1548 kvm_inject_nested_sync(vcpu, esr);
1551 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1553 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1554 u64 esr = kvm_vcpu_get_esr(vcpu);
1561 if (kvm_vcpu_abt_issea(vcpu))
1562 return kvm_handle_guest_sea(vcpu);
1565 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr);
1574 atomic_inc(&vcpu->kvm->arch.vncr_tlb_count);
1577 ret = kvm_translate_vncr(vcpu, &is_gmem);
1580 atomic_dec(&vcpu->kvm->arch.vncr_tlb_count);
1612 kvm_inject_nested_sync(vcpu, esr);
1615 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr;
1617 handle_vncr_perm(vcpu);
1624 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1626 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1630 guard(read_lock)(&vcpu->kvm->mmu_lock);
1636 if (is_hyp_ctxt(vcpu))
1649 if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr))
1652 if (read_vncr_el2(vcpu) != vt->gva)
1655 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid)
1857 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu,
1862 resx = kvm_get_sysreg_resx(vcpu->kvm, sr);
1878 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
1880 struct kvm *kvm = vcpu->kvm;
2016 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0);
2021 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
2023 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) {
2024 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
2026 write_lock(&vcpu->kvm->mmu_lock);
2031 write_unlock(&vcpu->kvm->mmu_lock);
2034 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
2035 kvm_map_l1_vncr(vcpu);
2038 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
2039 kvm_inject_nested_irq(vcpu);
2050 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
2052 if (!vcpu_has_nv(vcpu))
2055 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
2056 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
2059 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
2061 unsigned long *hcr = vcpu_hcr(vcpu);
2063 if (!vcpu_has_nv(vcpu))
2071 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
2078 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
2079 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
2097 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu)
2099 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
2101 if (is_nested_ctxt(vcpu))
2102 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
2112 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;