1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARM64_KVM_NESTED_H 3 #define __ARM64_KVM_NESTED_H 4 5 #include <linux/bitfield.h> 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_emulate.h> 8 #include <asm/kvm_pgtable.h> 9 10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) 11 { 12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) && 13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); 15 } 16 17 /* Translation helpers from non-VHE EL2 to EL1 */ 18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) 19 { 20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; 21 } 22 23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 24 { 25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ 26 ((tcr & TCR_EL2_DS) ? TCR_DS : 0) | 27 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | 28 tcr_el2_ps_to_tcr_el1_ips(tcr) | 29 (tcr & TCR_EL2_TG0_MASK) | 30 (tcr & TCR_EL2_ORGN0_MASK) | 31 (tcr & TCR_EL2_IRGN0_MASK) | 32 (tcr & TCR_EL2_T0SZ_MASK); 33 } 34 35 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 36 { 37 u64 cpacr_el1 = CPACR_EL1_RES1; 38 39 if (cptr_el2 & CPTR_EL2_TTA) 40 cpacr_el1 |= CPACR_EL1_TTA; 41 if (!(cptr_el2 & CPTR_EL2_TFP)) 42 cpacr_el1 |= CPACR_EL1_FPEN; 43 if (!(cptr_el2 & CPTR_EL2_TZ)) 44 cpacr_el1 |= CPACR_EL1_ZEN; 45 46 cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM); 47 48 return cpacr_el1; 49 } 50 51 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) 52 { 53 /* Only preserve the minimal set of bits we support */ 54 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | 55 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); 56 val |= SCTLR_EL1_RES1; 57 58 return val; 59 } 60 61 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 62 { 63 /* Clear the ASID field */ 64 return ttbr0 & ~GENMASK_ULL(63, 48); 65 } 66 67 extern bool forward_smc_trap(struct kvm_vcpu *vcpu); 68 extern bool forward_debug_exception(struct kvm_vcpu *vcpu); 69 extern int kvm_init_nested(struct kvm *kvm); 70 extern void kvm_destroy_nested(struct kvm *kvm); 71 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); 72 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); 73 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); 74 75 union tlbi_info; 76 77 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 78 const union tlbi_info *info, 79 void (*)(struct kvm_s2_mmu *, 80 const union tlbi_info *)); 81 extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, 82 gpa_t nested_ipa, size_t map_size, 83 struct kvm_guest_s2_mapping *mapping); 84 extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, 85 gpa_t nipa, size_t size); 86 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 87 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); 88 89 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu); 90 extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu); 91 extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu); 92 93 extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu); 94 95 struct kvm_s2_trans { 96 phys_addr_t output; 97 unsigned long block_size; 98 bool writable; 99 bool readable; 100 int level; 101 u32 esr; 102 u64 desc; 103 }; 104 105 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans) 106 { 107 return trans->output; 108 } 109 110 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans) 111 { 112 return trans->block_size; 113 } 114 115 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans) 116 { 117 return trans->esr; 118 } 119 120 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans) 121 { 122 return trans->readable; 123 } 124 125 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) 126 { 127 return trans->writable; 128 } 129 130 static inline bool kvm_has_xnx(struct kvm *kvm) 131 { 132 return cpus_have_final_cap(ARM64_HAS_XNX) && 133 kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP); 134 } 135 136 static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans) 137 { 138 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc); 139 140 if (!kvm_has_xnx(kvm)) 141 xn &= 0b10; 142 143 switch (xn) { 144 case 0b00: 145 case 0b01: 146 return true; 147 default: 148 return false; 149 } 150 } 151 152 static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans) 153 { 154 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc); 155 156 if (!kvm_has_xnx(kvm)) 157 xn &= 0b10; 158 159 switch (xn) { 160 case 0b00: 161 case 0b11: 162 return true; 163 default: 164 return false; 165 } 166 } 167 168 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 169 struct kvm_s2_trans *result); 170 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, 171 struct kvm_s2_trans *trans); 172 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); 173 extern void kvm_nested_s2_wp(struct kvm *kvm); 174 extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, 175 size_t unmap_size, bool may_block); 176 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); 177 extern void kvm_nested_s2_flush(struct kvm *kvm); 178 179 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val); 180 181 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr) 182 { 183 struct kvm *kvm = vpcu->kvm; 184 u8 CRm = sys_reg_CRm(instr); 185 186 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 187 sys_reg_Op1(instr) == TLBI_Op1_EL1)) 188 return false; 189 190 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 191 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 192 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 193 return false; 194 195 if (CRm == TLBI_CRm_nROS && 196 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 197 return false; 198 199 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 200 CRm == TLBI_CRm_RNS) && 201 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 202 return false; 203 204 return true; 205 } 206 207 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr) 208 { 209 struct kvm *kvm = vpcu->kvm; 210 u8 CRm = sys_reg_CRm(instr); 211 212 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 213 sys_reg_Op1(instr) == TLBI_Op1_EL2)) 214 return false; 215 216 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 217 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 218 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 219 return false; 220 221 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS) 222 return false; 223 224 if (CRm == TLBI_CRm_nROS && 225 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 226 return false; 227 228 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 229 CRm == TLBI_CRm_RNS) && 230 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 231 return false; 232 233 return true; 234 } 235 236 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu); 237 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val); 238 239 #ifdef CONFIG_ARM64_PTR_AUTH 240 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr); 241 #else 242 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 243 { 244 /* We really should never execute this... */ 245 WARN_ON_ONCE(1); 246 *elr = 0xbad9acc0debadbad; 247 return false; 248 } 249 #endif 250 251 #define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0) 252 253 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) 254 { 255 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level); 256 } 257 258 /* Adjust alignment for the contiguous bit as per StageOA() */ 259 #define contiguous_bit_shift(d, wi, l) \ 260 ({ \ 261 u8 shift = 0; \ 262 \ 263 if ((d) & PTE_CONT) { \ 264 switch (BIT((wi)->pgshift)) { \ 265 case SZ_4K: \ 266 shift = 4; \ 267 break; \ 268 case SZ_16K: \ 269 shift = (l) == 2 ? 5 : 7; \ 270 break; \ 271 case SZ_64K: \ 272 shift = 5; \ 273 break; \ 274 } \ 275 } \ 276 \ 277 shift; \ 278 }) 279 280 static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) 281 { 282 u64 base, tg, num, scale; 283 int shift; 284 285 tg = FIELD_GET(GENMASK(47, 46), val); 286 287 switch(tg) { 288 case 1: 289 shift = 12; 290 break; 291 case 2: 292 shift = 14; 293 break; 294 case 3: 295 default: /* IMPDEF: handle tg==0 as 64k */ 296 shift = 16; 297 break; 298 } 299 300 base = (val & GENMASK(36, 0)) << shift; 301 302 /* 303 * We only deal with at most 48bit VA/IPA, so 48 is where we 304 * sign-extend from. Should we support FEAT_L{VP}A* at some point, 305 * this will need to be revisited. 306 */ 307 base = (u64)sign_extend64(base, 48); 308 309 if (asid) 310 *asid = FIELD_GET(TLBIR_ASID_MASK, val); 311 312 scale = FIELD_GET(GENMASK(45, 44), val); 313 num = FIELD_GET(GENMASK(43, 39), val); 314 *range = __TLBI_RANGE_PAGES(num, scale) << shift; 315 316 /* Cap the range to the correct half of the address space */ 317 if (!(base & BIT(48))) 318 *range = min(*range, (BIT(48) - base)); 319 else 320 *range = min(*range, ~base + 1); 321 322 return base; 323 } 324 325 static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit) 326 { 327 switch (ps) { 328 case 0: return 32; 329 case 1: return 36; 330 case 2: return 40; 331 case 3: return 42; 332 case 4: return 44; 333 case 5: return 48; 334 case 6: if (pa52bit) 335 return 52; 336 fallthrough; 337 default: 338 return 48; 339 } 340 } 341 342 enum trans_regime { 343 TR_EL10, 344 TR_EL20, 345 TR_EL2, 346 }; 347 348 struct s1_walk_info; 349 350 struct s1_walk_context { 351 struct s1_walk_info *wi; 352 u64 table_ipa; 353 int level; 354 }; 355 356 struct s1_walk_filter { 357 int (*fn)(struct s1_walk_context *, void *); 358 void *priv; 359 }; 360 361 struct s1_walk_info { 362 struct s1_walk_filter *filter; 363 u64 baddr; 364 enum trans_regime regime; 365 unsigned int max_oa_bits; 366 unsigned int pgshift; 367 unsigned int txsz; 368 int sl; 369 u8 sh; 370 bool as_el0; 371 bool hpd; 372 bool e0poe; 373 bool poe; 374 bool pan; 375 bool be; 376 bool s2; 377 bool pa52bit; 378 bool ha; 379 }; 380 381 struct s1_walk_result { 382 union { 383 struct { 384 u64 desc; 385 u64 pa; 386 s8 level; 387 u8 APTable; 388 bool nG; 389 u16 asid; 390 bool UXNTable; 391 bool PXNTable; 392 bool uwxn; 393 bool uov; 394 bool ur; 395 bool uw; 396 bool ux; 397 bool pwxn; 398 bool pov; 399 bool pr; 400 bool pw; 401 bool px; 402 }; 403 struct { 404 u8 fst; 405 bool ptw; 406 bool s2; 407 }; 408 }; 409 bool failed; 410 }; 411 412 #define S1_MMU_DISABLED (-127) 413 414 static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) 415 { 416 wr->fst = fst; 417 wr->ptw = s1ptw; 418 wr->s2 = s1ptw; 419 wr->failed = true; 420 } 421 422 static inline bool s1_walk_translated(struct s1_walk_result *wr) 423 { 424 return wr->level != S1_MMU_DISABLED; 425 } 426 427 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 428 struct s1_walk_result *wr, u64 va); 429 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, 430 int *level); 431 432 /* VNCR management */ 433 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu); 434 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu); 435 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); 436 437 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime); 438 439 #define vncr_fixmap(c) \ 440 ({ \ 441 u32 __c = (c); \ 442 BUG_ON(__c >= NR_CPUS); \ 443 (FIX_VNCR - __c); \ 444 }) 445 446 int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new); 447 448 #endif /* __ARM64_KVM_NESTED_H */ 449