1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Fault injection for both 32 and 64bit guests. 4 * 5 * Copyright (C) 2012,2013 - ARM Ltd 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 * 8 * Based on arch/arm/kvm/emulate.c 9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 10 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 11 */ 12 13 #include <linux/kvm_host.h> 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_nested.h> 16 #include <asm/esr.h> 17 18 static unsigned int exception_target_el(struct kvm_vcpu *vcpu) 19 { 20 /* If not nesting, EL1 is the only possible exception target */ 21 if (likely(!vcpu_has_nv(vcpu))) 22 return PSR_MODE_EL1h; 23 24 /* 25 * With NV, we need to pick between EL1 and EL2. Note that we 26 * never deal with a nesting exception here, hence never 27 * changing context, and the exception itself can be delayed 28 * until the next entry. 29 */ 30 switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) { 31 case PSR_MODE_EL2h: 32 case PSR_MODE_EL2t: 33 return PSR_MODE_EL2h; 34 case PSR_MODE_EL1h: 35 case PSR_MODE_EL1t: 36 return PSR_MODE_EL1h; 37 case PSR_MODE_EL0t: 38 return vcpu_el2_tge_is_set(vcpu) ? PSR_MODE_EL2h : PSR_MODE_EL1h; 39 default: 40 BUG(); 41 } 42 } 43 44 static enum vcpu_sysreg exception_esr_elx(struct kvm_vcpu *vcpu) 45 { 46 if (exception_target_el(vcpu) == PSR_MODE_EL2h) 47 return ESR_EL2; 48 49 return ESR_EL1; 50 } 51 52 static enum vcpu_sysreg exception_far_elx(struct kvm_vcpu *vcpu) 53 { 54 if (exception_target_el(vcpu) == PSR_MODE_EL2h) 55 return FAR_EL2; 56 57 return FAR_EL1; 58 } 59 60 static void pend_sync_exception(struct kvm_vcpu *vcpu) 61 { 62 if (exception_target_el(vcpu) == PSR_MODE_EL1h) 63 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); 64 else 65 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC); 66 } 67 68 static void pend_serror_exception(struct kvm_vcpu *vcpu) 69 { 70 if (exception_target_el(vcpu) == PSR_MODE_EL1h) 71 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SERR); 72 else 73 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR); 74 } 75 76 static bool __effective_sctlr2_bit(struct kvm_vcpu *vcpu, unsigned int idx) 77 { 78 u64 sctlr2; 79 80 if (!kvm_has_sctlr2(vcpu->kvm)) 81 return false; 82 83 if (is_nested_ctxt(vcpu) && 84 !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_SCTLR2En)) 85 return false; 86 87 if (exception_target_el(vcpu) == PSR_MODE_EL1h) 88 sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL1); 89 else 90 sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL2); 91 92 return sctlr2 & BIT(idx); 93 } 94 95 static bool effective_sctlr2_ease(struct kvm_vcpu *vcpu) 96 { 97 return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_EASE_SHIFT); 98 } 99 100 static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu) 101 { 102 return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT); 103 } 104 105 static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr) 106 { 107 unsigned long cpsr = *vcpu_cpsr(vcpu); 108 bool is_aarch32 = vcpu_mode_is_32bit(vcpu); 109 u64 esr = 0, fsc; 110 int level; 111 112 /* 113 * If injecting an abort from a failed S1PTW, rewalk the S1 PTs to 114 * find the failing level. If we can't find it, assume the error was 115 * transient and restart without changing the state. 116 */ 117 if (kvm_vcpu_abt_iss1tw(vcpu)) { 118 u64 hpfar = kvm_vcpu_get_fault_ipa(vcpu); 119 int ret; 120 121 if (hpfar == INVALID_GPA) 122 return; 123 124 ret = __kvm_find_s1_desc_level(vcpu, addr, hpfar, &level); 125 if (ret) 126 return; 127 128 WARN_ON_ONCE(level < -1 || level > 3); 129 fsc = ESR_ELx_FSC_SEA_TTW(level); 130 } else { 131 fsc = ESR_ELx_FSC_EXTABT; 132 } 133 134 /* This delight is brought to you by FEAT_DoubleFault2. */ 135 if (effective_sctlr2_ease(vcpu)) 136 pend_serror_exception(vcpu); 137 else 138 pend_sync_exception(vcpu); 139 140 /* 141 * Build an {i,d}abort, depending on the level. 142 * Report an external synchronous abort. 143 */ 144 esr |= ESR_ELx_IL; 145 146 /* 147 * Here, the guest runs in AArch64 mode when in EL1. If we get 148 * an AArch32 fault, it means we managed to trap an EL0 fault. 149 */ 150 if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) 151 esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); 152 else 153 esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); 154 155 if (!is_iabt) 156 esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT; 157 158 esr |= fsc; 159 160 vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu)); 161 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); 162 } 163 164 void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr) 165 { 166 pend_sync_exception(vcpu); 167 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); 168 } 169 170 static void inject_undef64(struct kvm_vcpu *vcpu) 171 { 172 u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; 173 174 kvm_inject_sync(vcpu, esr); 175 } 176 177 #define DFSR_FSC_EXTABT_LPAE 0x10 178 #define DFSR_FSC_EXTABT_nLPAE 0x08 179 #define DFSR_LPAE BIT(9) 180 #define TTBCR_EAE BIT(31) 181 182 static void inject_undef32(struct kvm_vcpu *vcpu) 183 { 184 kvm_pend_exception(vcpu, EXCEPT_AA32_UND); 185 } 186 187 /* 188 * Modelled after TakeDataAbortException() and TakePrefetchAbortException 189 * pseudocode. 190 */ 191 static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr) 192 { 193 u64 far; 194 u32 fsr; 195 196 /* Give the guest an IMPLEMENTATION DEFINED exception */ 197 if (vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE) { 198 fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE; 199 } else { 200 /* no need to shuffle FS[4] into DFSR[10] as it's 0 */ 201 fsr = DFSR_FSC_EXTABT_nLPAE; 202 } 203 204 far = vcpu_read_sys_reg(vcpu, FAR_EL1); 205 206 if (is_pabt) { 207 kvm_pend_exception(vcpu, EXCEPT_AA32_IABT); 208 far &= GENMASK(31, 0); 209 far |= (u64)addr << 32; 210 vcpu_write_sys_reg(vcpu, fsr, IFSR32_EL2); 211 } else { /* !iabt */ 212 kvm_pend_exception(vcpu, EXCEPT_AA32_DABT); 213 far &= GENMASK(63, 32); 214 far |= addr; 215 vcpu_write_sys_reg(vcpu, fsr, ESR_EL1); 216 } 217 218 vcpu_write_sys_reg(vcpu, far, FAR_EL1); 219 } 220 221 static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) 222 { 223 if (vcpu_el1_is_32bit(vcpu)) 224 inject_abt32(vcpu, iabt, addr); 225 else 226 inject_abt64(vcpu, iabt, addr); 227 } 228 229 static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu) 230 { 231 if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA)) 232 return true; 233 234 if (!vcpu_mode_priv(vcpu)) 235 return false; 236 237 return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && 238 (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); 239 } 240 241 int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) 242 { 243 lockdep_assert_held(&vcpu->mutex); 244 245 if (is_nested_ctxt(vcpu) && kvm_sea_target_is_el2(vcpu)) 246 return kvm_inject_nested_sea(vcpu, iabt, addr); 247 248 __kvm_inject_sea(vcpu, iabt, addr); 249 return 1; 250 } 251 252 static int kvm_inject_nested_excl_atomic(struct kvm_vcpu *vcpu, u64 addr) 253 { 254 u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_DABT_LOW) | 255 FIELD_PREP(ESR_ELx_FSC, ESR_ELx_FSC_EXCL_ATOMIC) | 256 ESR_ELx_IL; 257 258 vcpu_write_sys_reg(vcpu, addr, FAR_EL2); 259 return kvm_inject_nested_sync(vcpu, esr); 260 } 261 262 /** 263 * kvm_inject_dabt_excl_atomic - inject a data abort for unsupported exclusive 264 * or atomic access 265 * @vcpu: The VCPU to receive the data abort 266 * @addr: The address to report in the DFAR 267 * 268 * It is assumed that this code is called from the VCPU thread and that the 269 * VCPU therefore is not currently executing guest code. 270 */ 271 int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr) 272 { 273 u64 esr; 274 275 if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM)) 276 return kvm_inject_nested_excl_atomic(vcpu, addr); 277 278 __kvm_inject_sea(vcpu, false, addr); 279 esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu)); 280 esr &= ~ESR_ELx_FSC; 281 esr |= ESR_ELx_FSC_EXCL_ATOMIC; 282 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); 283 return 1; 284 } 285 286 void kvm_inject_size_fault(struct kvm_vcpu *vcpu) 287 { 288 unsigned long addr, esr; 289 290 addr = kvm_vcpu_get_fault_ipa(vcpu); 291 addr |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu)); 292 293 __kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr); 294 295 /* 296 * If AArch64 or LPAE, set FSC to 0 to indicate an Address 297 * Size Fault at level 0, as if exceeding PARange. 298 * 299 * Non-LPAE guests will only get the external abort, as there 300 * is no way to describe the ASF. 301 */ 302 if (vcpu_el1_is_32bit(vcpu) && 303 !(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE)) 304 return; 305 306 esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu)); 307 esr &= ~GENMASK_ULL(5, 0); 308 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); 309 } 310 311 /** 312 * kvm_inject_undefined - inject an undefined instruction into the guest 313 * @vcpu: The vCPU in which to inject the exception 314 * 315 * It is assumed that this code is called from the VCPU thread and that the 316 * VCPU therefore is not currently executing guest code. 317 */ 318 void kvm_inject_undefined(struct kvm_vcpu *vcpu) 319 { 320 if (vcpu_el1_is_32bit(vcpu)) 321 inject_undef32(vcpu); 322 else 323 inject_undef64(vcpu); 324 } 325 326 static bool serror_is_masked(struct kvm_vcpu *vcpu) 327 { 328 return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu); 329 } 330 331 static bool kvm_serror_target_is_el2(struct kvm_vcpu *vcpu) 332 { 333 if (is_hyp_ctxt(vcpu) || vcpu_el2_amo_is_set(vcpu)) 334 return true; 335 336 if (!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA)) 337 return false; 338 339 /* 340 * In another example where FEAT_DoubleFault2 is entirely backwards, 341 * "masked" as it relates to the routing effects of HCRX_EL2.TMEA 342 * doesn't consider SCTLR2_EL1.NMEA. That is to say, even if EL1 asked 343 * for non-maskable SErrors, the EL2 bit takes priority if A is set. 344 */ 345 if (vcpu_mode_priv(vcpu)) 346 return *vcpu_cpsr(vcpu) & PSR_A_BIT; 347 348 /* 349 * Otherwise SErrors are considered unmasked when taken from EL0 and 350 * NMEA is set. 351 */ 352 return serror_is_masked(vcpu); 353 } 354 355 static bool kvm_serror_undeliverable_at_el2(struct kvm_vcpu *vcpu) 356 { 357 return !(vcpu_el2_tge_is_set(vcpu) || vcpu_el2_amo_is_set(vcpu)); 358 } 359 360 int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) 361 { 362 lockdep_assert_held(&vcpu->mutex); 363 364 if (is_nested_ctxt(vcpu) && kvm_serror_target_is_el2(vcpu)) 365 return kvm_inject_nested_serror(vcpu, esr); 366 367 if (vcpu_is_el2(vcpu) && kvm_serror_undeliverable_at_el2(vcpu)) { 368 vcpu_set_vsesr(vcpu, esr); 369 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); 370 return 1; 371 } 372 373 /* 374 * Emulate the exception entry if SErrors are unmasked. This is useful if 375 * the vCPU is in a nested context w/ vSErrors enabled then we've already 376 * delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2, 377 * VDISR_EL2) to the guest hypervisor. 378 * 379 * As we're emulating the SError injection we need to explicitly populate 380 * ESR_ELx.EC because hardware will not do it on our behalf. 381 */ 382 if (!serror_is_masked(vcpu)) { 383 pend_serror_exception(vcpu); 384 esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; 385 vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); 386 return 1; 387 } 388 389 vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK); 390 *vcpu_hcr(vcpu) |= HCR_VSE; 391 return 1; 392 } 393