1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARM64_KVM_NESTED_H 3 #define __ARM64_KVM_NESTED_H 4 5 #include <linux/bitfield.h> 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_emulate.h> 8 #include <asm/kvm_pgtable.h> 9 10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) 11 { 12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) && 13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); 15 } 16 17 /* Translation helpers from non-VHE EL2 to EL1 */ 18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) 19 { 20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; 21 } 22 23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 24 { 25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ 26 ((tcr & TCR_EL2_DS) ? TCR_DS : 0) | 27 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | 28 tcr_el2_ps_to_tcr_el1_ips(tcr) | 29 (tcr & TCR_EL2_TG0_MASK) | 30 (tcr & TCR_EL2_ORGN0_MASK) | 31 (tcr & TCR_EL2_IRGN0_MASK) | 32 (tcr & TCR_EL2_T0SZ_MASK); 33 } 34 35 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 36 { 37 u64 cpacr_el1 = CPACR_EL1_RES1; 38 39 if (cptr_el2 & CPTR_EL2_TTA) 40 cpacr_el1 |= CPACR_EL1_TTA; 41 if (!(cptr_el2 & CPTR_EL2_TFP)) 42 cpacr_el1 |= CPACR_EL1_FPEN; 43 if (!(cptr_el2 & CPTR_EL2_TZ)) 44 cpacr_el1 |= CPACR_EL1_ZEN; 45 46 cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM); 47 48 return cpacr_el1; 49 } 50 51 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) 52 { 53 /* Only preserve the minimal set of bits we support */ 54 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | 55 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); 56 val |= SCTLR_EL1_RES1; 57 58 return val; 59 } 60 61 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 62 { 63 /* Clear the ASID field */ 64 return ttbr0 & ~GENMASK_ULL(63, 48); 65 } 66 67 extern bool forward_smc_trap(struct kvm_vcpu *vcpu); 68 extern bool forward_debug_exception(struct kvm_vcpu *vcpu); 69 extern void kvm_init_nested(struct kvm *kvm); 70 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); 71 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); 72 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); 73 74 union tlbi_info; 75 76 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 77 const union tlbi_info *info, 78 void (*)(struct kvm_s2_mmu *, 79 const union tlbi_info *)); 80 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 81 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); 82 83 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu); 84 extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu); 85 extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu); 86 87 extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu); 88 89 struct kvm_s2_trans { 90 phys_addr_t output; 91 unsigned long block_size; 92 bool writable; 93 bool readable; 94 int level; 95 u32 esr; 96 u64 desc; 97 }; 98 99 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans) 100 { 101 return trans->output; 102 } 103 104 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans) 105 { 106 return trans->block_size; 107 } 108 109 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans) 110 { 111 return trans->esr; 112 } 113 114 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans) 115 { 116 return trans->readable; 117 } 118 119 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) 120 { 121 return trans->writable; 122 } 123 124 static inline bool kvm_has_xnx(struct kvm *kvm) 125 { 126 return cpus_have_final_cap(ARM64_HAS_XNX) && 127 kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP); 128 } 129 130 static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans) 131 { 132 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc); 133 134 if (!kvm_has_xnx(kvm)) 135 xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10); 136 137 switch (xn) { 138 case 0b00: 139 case 0b01: 140 return true; 141 default: 142 return false; 143 } 144 } 145 146 static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans) 147 { 148 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc); 149 150 if (!kvm_has_xnx(kvm)) 151 xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10); 152 153 switch (xn) { 154 case 0b00: 155 case 0b11: 156 return true; 157 default: 158 return false; 159 } 160 } 161 162 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 163 struct kvm_s2_trans *result); 164 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, 165 struct kvm_s2_trans *trans); 166 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); 167 extern void kvm_nested_s2_wp(struct kvm *kvm); 168 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); 169 extern void kvm_nested_s2_flush(struct kvm *kvm); 170 171 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val); 172 173 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr) 174 { 175 struct kvm *kvm = vpcu->kvm; 176 u8 CRm = sys_reg_CRm(instr); 177 178 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 179 sys_reg_Op1(instr) == TLBI_Op1_EL1)) 180 return false; 181 182 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 183 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 184 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 185 return false; 186 187 if (CRm == TLBI_CRm_nROS && 188 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 189 return false; 190 191 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 192 CRm == TLBI_CRm_RNS) && 193 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 194 return false; 195 196 return true; 197 } 198 199 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr) 200 { 201 struct kvm *kvm = vpcu->kvm; 202 u8 CRm = sys_reg_CRm(instr); 203 204 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 205 sys_reg_Op1(instr) == TLBI_Op1_EL2)) 206 return false; 207 208 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 209 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 210 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 211 return false; 212 213 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS) 214 return false; 215 216 if (CRm == TLBI_CRm_nROS && 217 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 218 return false; 219 220 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 221 CRm == TLBI_CRm_RNS) && 222 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 223 return false; 224 225 return true; 226 } 227 228 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu); 229 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val); 230 231 #ifdef CONFIG_ARM64_PTR_AUTH 232 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr); 233 #else 234 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 235 { 236 /* We really should never execute this... */ 237 WARN_ON_ONCE(1); 238 *elr = 0xbad9acc0debadbad; 239 return false; 240 } 241 #endif 242 243 #define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0) 244 245 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) 246 { 247 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level); 248 } 249 250 /* Adjust alignment for the contiguous bit as per StageOA() */ 251 #define contiguous_bit_shift(d, wi, l) \ 252 ({ \ 253 u8 shift = 0; \ 254 \ 255 if ((d) & PTE_CONT) { \ 256 switch (BIT((wi)->pgshift)) { \ 257 case SZ_4K: \ 258 shift = 4; \ 259 break; \ 260 case SZ_16K: \ 261 shift = (l) == 2 ? 5 : 7; \ 262 break; \ 263 case SZ_64K: \ 264 shift = 5; \ 265 break; \ 266 } \ 267 } \ 268 \ 269 shift; \ 270 }) 271 272 static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) 273 { 274 u64 base, tg, num, scale; 275 int shift; 276 277 tg = FIELD_GET(GENMASK(47, 46), val); 278 279 switch(tg) { 280 case 1: 281 shift = 12; 282 break; 283 case 2: 284 shift = 14; 285 break; 286 case 3: 287 default: /* IMPDEF: handle tg==0 as 64k */ 288 shift = 16; 289 break; 290 } 291 292 base = (val & GENMASK(36, 0)) << shift; 293 294 if (asid) 295 *asid = FIELD_GET(TLBIR_ASID_MASK, val); 296 297 scale = FIELD_GET(GENMASK(45, 44), val); 298 num = FIELD_GET(GENMASK(43, 39), val); 299 *range = __TLBI_RANGE_PAGES(num, scale) << shift; 300 301 return base; 302 } 303 304 static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit) 305 { 306 switch (ps) { 307 case 0: return 32; 308 case 1: return 36; 309 case 2: return 40; 310 case 3: return 42; 311 case 4: return 44; 312 case 5: return 48; 313 case 6: if (pa52bit) 314 return 52; 315 fallthrough; 316 default: 317 return 48; 318 } 319 } 320 321 enum trans_regime { 322 TR_EL10, 323 TR_EL20, 324 TR_EL2, 325 }; 326 327 struct s1_walk_info; 328 329 struct s1_walk_context { 330 struct s1_walk_info *wi; 331 u64 table_ipa; 332 int level; 333 }; 334 335 struct s1_walk_filter { 336 int (*fn)(struct s1_walk_context *, void *); 337 void *priv; 338 }; 339 340 struct s1_walk_info { 341 struct s1_walk_filter *filter; 342 u64 baddr; 343 enum trans_regime regime; 344 unsigned int max_oa_bits; 345 unsigned int pgshift; 346 unsigned int txsz; 347 int sl; 348 u8 sh; 349 bool as_el0; 350 bool hpd; 351 bool e0poe; 352 bool poe; 353 bool pan; 354 bool be; 355 bool s2; 356 bool pa52bit; 357 bool ha; 358 }; 359 360 struct s1_walk_result { 361 union { 362 struct { 363 u64 desc; 364 u64 pa; 365 s8 level; 366 u8 APTable; 367 bool nG; 368 u16 asid; 369 bool UXNTable; 370 bool PXNTable; 371 bool uwxn; 372 bool uov; 373 bool ur; 374 bool uw; 375 bool ux; 376 bool pwxn; 377 bool pov; 378 bool pr; 379 bool pw; 380 bool px; 381 }; 382 struct { 383 u8 fst; 384 bool ptw; 385 bool s2; 386 }; 387 }; 388 bool failed; 389 }; 390 391 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 392 struct s1_walk_result *wr, u64 va); 393 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, 394 int *level); 395 396 /* VNCR management */ 397 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu); 398 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu); 399 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); 400 401 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime); 402 403 #define vncr_fixmap(c) \ 404 ({ \ 405 u32 __c = (c); \ 406 BUG_ON(__c >= NR_CPUS); \ 407 (FIX_VNCR - __c); \ 408 }) 409 410 int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new); 411 412 #endif /* __ARM64_KVM_NESTED_H */ 413