1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARM64_KVM_NESTED_H 3 #define __ARM64_KVM_NESTED_H 4 5 #include <linux/bitfield.h> 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_emulate.h> 8 #include <asm/kvm_pgtable.h> 9 10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) 11 { 12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) && 13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); 15 } 16 17 /* Translation helpers from non-VHE EL2 to EL1 */ 18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) 19 { 20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; 21 } 22 23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 24 { 25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ 26 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | 27 tcr_el2_ps_to_tcr_el1_ips(tcr) | 28 (tcr & TCR_EL2_TG0_MASK) | 29 (tcr & TCR_EL2_ORGN0_MASK) | 30 (tcr & TCR_EL2_IRGN0_MASK) | 31 (tcr & TCR_EL2_T0SZ_MASK); 32 } 33 34 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 35 { 36 u64 cpacr_el1 = 0; 37 38 if (cptr_el2 & CPTR_EL2_TTA) 39 cpacr_el1 |= CPACR_ELx_TTA; 40 if (!(cptr_el2 & CPTR_EL2_TFP)) 41 cpacr_el1 |= CPACR_ELx_FPEN; 42 if (!(cptr_el2 & CPTR_EL2_TZ)) 43 cpacr_el1 |= CPACR_ELx_ZEN; 44 45 return cpacr_el1; 46 } 47 48 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) 49 { 50 /* Only preserve the minimal set of bits we support */ 51 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | 52 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); 53 val |= SCTLR_EL1_RES1; 54 55 return val; 56 } 57 58 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 59 { 60 /* Clear the ASID field */ 61 return ttbr0 & ~GENMASK_ULL(63, 48); 62 } 63 64 extern bool forward_smc_trap(struct kvm_vcpu *vcpu); 65 extern void kvm_init_nested(struct kvm *kvm); 66 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); 67 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); 68 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); 69 70 union tlbi_info; 71 72 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 73 const union tlbi_info *info, 74 void (*)(struct kvm_s2_mmu *, 75 const union tlbi_info *)); 76 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 77 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); 78 79 struct kvm_s2_trans { 80 phys_addr_t output; 81 unsigned long block_size; 82 bool writable; 83 bool readable; 84 int level; 85 u32 esr; 86 u64 upper_attr; 87 }; 88 89 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans) 90 { 91 return trans->output; 92 } 93 94 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans) 95 { 96 return trans->block_size; 97 } 98 99 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans) 100 { 101 return trans->esr; 102 } 103 104 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans) 105 { 106 return trans->readable; 107 } 108 109 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) 110 { 111 return trans->writable; 112 } 113 114 static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans) 115 { 116 return !(trans->upper_attr & BIT(54)); 117 } 118 119 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 120 struct kvm_s2_trans *result); 121 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, 122 struct kvm_s2_trans *trans); 123 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); 124 extern void kvm_nested_s2_wp(struct kvm *kvm); 125 extern void kvm_nested_s2_unmap(struct kvm *kvm); 126 extern void kvm_nested_s2_flush(struct kvm *kvm); 127 128 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val); 129 130 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr) 131 { 132 struct kvm *kvm = vpcu->kvm; 133 u8 CRm = sys_reg_CRm(instr); 134 135 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 136 sys_reg_Op1(instr) == TLBI_Op1_EL1)) 137 return false; 138 139 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 140 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 141 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 142 return false; 143 144 if (CRm == TLBI_CRm_nROS && 145 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 146 return false; 147 148 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 149 CRm == TLBI_CRm_RNS) && 150 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 151 return false; 152 153 return true; 154 } 155 156 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr) 157 { 158 struct kvm *kvm = vpcu->kvm; 159 u8 CRm = sys_reg_CRm(instr); 160 161 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 162 sys_reg_Op1(instr) == TLBI_Op1_EL2)) 163 return false; 164 165 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 166 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 167 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 168 return false; 169 170 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS) 171 return false; 172 173 if (CRm == TLBI_CRm_nROS && 174 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 175 return false; 176 177 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 178 CRm == TLBI_CRm_RNS) && 179 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 180 return false; 181 182 return true; 183 } 184 185 int kvm_init_nv_sysregs(struct kvm *kvm); 186 187 #ifdef CONFIG_ARM64_PTR_AUTH 188 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr); 189 #else 190 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 191 { 192 /* We really should never execute this... */ 193 WARN_ON_ONCE(1); 194 *elr = 0xbad9acc0debadbad; 195 return false; 196 } 197 #endif 198 199 #define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0) 200 201 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) 202 { 203 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level); 204 } 205 206 #endif /* __ARM64_KVM_NESTED_H */ 207