1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Fault injection for both 32 and 64bit guests. 4 * 5 * Copyright (C) 2012,2013 - ARM Ltd 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 * 8 * Based on arch/arm/kvm/emulate.c 9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 10 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 11 */ 12 13 #include <hyp/adjust_pc.h> 14 #include <linux/kvm_host.h> 15 #include <asm/kvm_emulate.h> 16 #include <asm/kvm_mmu.h> 17 #include <asm/kvm_nested.h> 18 19 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) 20 #error Hypervisor code only! 21 #endif 22 23 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode, 24 u64 val) 25 { 26 if (has_vhe()) { 27 if (target_mode == PSR_MODE_EL1h) 28 vcpu_write_sys_reg(vcpu, val, SPSR_EL1); 29 else 30 vcpu_write_sys_reg(vcpu, val, SPSR_EL2); 31 } else { 32 __vcpu_assign_sys_reg(vcpu, SPSR_EL1, val); 33 } 34 } 35 36 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val) 37 { 38 if (has_vhe() && vcpu_get_flag(vcpu, SYSREGS_ON_CPU)) 39 write_sysreg(val, spsr_abt); 40 else 41 vcpu->arch.ctxt.spsr_abt = val; 42 } 43 44 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val) 45 { 46 if (has_vhe() && vcpu_get_flag(vcpu, SYSREGS_ON_CPU)) 47 write_sysreg(val, spsr_und); 48 else 49 vcpu->arch.ctxt.spsr_und = val; 50 } 51 52 /* 53 * This performs the exception entry at a given EL (@target_mode), stashing PC 54 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE. 55 * The EL passed to this function *must* be a non-secure, privileged mode with 56 * bit 0 being set (PSTATE.SP == 1). 57 * 58 * When an exception is taken, most PSTATE fields are left unchanged in the 59 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all 60 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx 61 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0. 62 * 63 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429. 64 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426. 65 * 66 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from 67 * MSB to LSB. 68 */ 69 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, 70 enum exception_type type) 71 { 72 unsigned long sctlr, vbar, old, new, mode; 73 u64 exc_offset; 74 75 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); 76 77 if (mode == target_mode) 78 exc_offset = CURRENT_EL_SP_ELx_VECTOR; 79 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode) 80 exc_offset = CURRENT_EL_SP_EL0_VECTOR; 81 else if (!(mode & PSR_MODE32_BIT)) 82 exc_offset = LOWER_EL_AArch64_VECTOR; 83 else 84 exc_offset = LOWER_EL_AArch32_VECTOR; 85 86 switch (target_mode) { 87 case PSR_MODE_EL1h: 88 vbar = vcpu_read_sys_reg(vcpu, VBAR_EL1); 89 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 90 vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); 91 break; 92 case PSR_MODE_EL2h: 93 vbar = vcpu_read_sys_reg(vcpu, VBAR_EL2); 94 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); 95 vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); 96 break; 97 default: 98 /* Don't do that */ 99 BUG(); 100 } 101 102 *vcpu_pc(vcpu) = vbar + exc_offset + type; 103 104 old = *vcpu_cpsr(vcpu); 105 new = 0; 106 107 new |= (old & PSR_N_BIT); 108 new |= (old & PSR_Z_BIT); 109 new |= (old & PSR_C_BIT); 110 new |= (old & PSR_V_BIT); 111 112 if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) 113 new |= PSR_TCO_BIT; 114 115 new |= (old & PSR_DIT_BIT); 116 117 // PSTATE.UAO is set to zero upon any exception to AArch64 118 // See ARM DDI 0487E.a, page D5-2579. 119 120 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0 121 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented 122 // See ARM DDI 0487E.a, page D5-2578. 123 new |= (old & PSR_PAN_BIT); 124 if (!(sctlr & SCTLR_EL1_SPAN)) 125 new |= PSR_PAN_BIT; 126 127 // PSTATE.SS is set to zero upon any exception to AArch64 128 // See ARM DDI 0487E.a, page D2-2452. 129 130 // PSTATE.IL is set to zero upon any exception to AArch64 131 // See ARM DDI 0487E.a, page D1-2306. 132 133 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64 134 // See ARM DDI 0487E.a, page D13-3258 135 if (sctlr & SCTLR_ELx_DSSBS) 136 new |= PSR_SSBS_BIT; 137 138 // PSTATE.BTYPE is set to zero upon any exception to AArch64 139 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294. 140 141 new |= PSR_D_BIT; 142 new |= PSR_A_BIT; 143 new |= PSR_I_BIT; 144 new |= PSR_F_BIT; 145 146 new |= target_mode; 147 148 *vcpu_cpsr(vcpu) = new; 149 __vcpu_write_spsr(vcpu, target_mode, old); 150 } 151 152 /* 153 * When an exception is taken, most CPSR fields are left unchanged in the 154 * handler. However, some are explicitly overridden (e.g. M[4:0]). 155 * 156 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with 157 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was 158 * obsoleted by the ARMv7 virtualization extensions and is RES0. 159 * 160 * For the SPSR layout seen from AArch32, see: 161 * - ARM DDI 0406C.d, page B1-1148 162 * - ARM DDI 0487E.a, page G8-6264 163 * 164 * For the SPSR_ELx layout for AArch32 seen from AArch64, see: 165 * - ARM DDI 0487E.a, page C5-426 166 * 167 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from 168 * MSB to LSB. 169 */ 170 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) 171 { 172 u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 173 unsigned long old, new; 174 175 old = *vcpu_cpsr(vcpu); 176 new = 0; 177 178 new |= (old & PSR_AA32_N_BIT); 179 new |= (old & PSR_AA32_Z_BIT); 180 new |= (old & PSR_AA32_C_BIT); 181 new |= (old & PSR_AA32_V_BIT); 182 new |= (old & PSR_AA32_Q_BIT); 183 184 // CPSR.IT[7:0] are set to zero upon any exception 185 // See ARM DDI 0487E.a, section G1.12.3 186 // See ARM DDI 0406C.d, section B1.8.3 187 188 new |= (old & PSR_AA32_DIT_BIT); 189 190 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception 191 // See ARM DDI 0487E.a, page G8-6244 192 if (sctlr & BIT(31)) 193 new |= PSR_AA32_SSBS_BIT; 194 195 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0 196 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented 197 // See ARM DDI 0487E.a, page G8-6246 198 new |= (old & PSR_AA32_PAN_BIT); 199 if (!(sctlr & BIT(23))) 200 new |= PSR_AA32_PAN_BIT; 201 202 // SS does not exist in AArch32, so ignore 203 204 // CPSR.IL is set to zero upon any exception 205 // See ARM DDI 0487E.a, page G1-5527 206 207 new |= (old & PSR_AA32_GE_MASK); 208 209 // CPSR.IT[7:0] are set to zero upon any exception 210 // See prior comment above 211 212 // CPSR.E is set to SCTLR.EE upon any exception 213 // See ARM DDI 0487E.a, page G8-6245 214 // See ARM DDI 0406C.d, page B4-1701 215 if (sctlr & BIT(25)) 216 new |= PSR_AA32_E_BIT; 217 218 // CPSR.A is unchanged upon an exception to Undefined, Supervisor 219 // CPSR.A is set upon an exception to other modes 220 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 221 // See ARM DDI 0406C.d, page B1-1182 222 new |= (old & PSR_AA32_A_BIT); 223 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC) 224 new |= PSR_AA32_A_BIT; 225 226 // CPSR.I is set upon any exception 227 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 228 // See ARM DDI 0406C.d, page B1-1182 229 new |= PSR_AA32_I_BIT; 230 231 // CPSR.F is set upon an exception to FIQ 232 // CPSR.F is unchanged upon an exception to other modes 233 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 234 // See ARM DDI 0406C.d, page B1-1182 235 new |= (old & PSR_AA32_F_BIT); 236 if (mode == PSR_AA32_MODE_FIQ) 237 new |= PSR_AA32_F_BIT; 238 239 // CPSR.T is set to SCTLR.TE upon any exception 240 // See ARM DDI 0487E.a, page G8-5514 241 // See ARM DDI 0406C.d, page B1-1181 242 if (sctlr & BIT(30)) 243 new |= PSR_AA32_T_BIT; 244 245 new |= mode; 246 247 return new; 248 } 249 250 /* 251 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10. 252 */ 253 static const u8 return_offsets[8][2] = { 254 [0] = { 0, 0 }, /* Reset, unused */ 255 [1] = { 4, 2 }, /* Undefined */ 256 [2] = { 0, 0 }, /* SVC, unused */ 257 [3] = { 4, 4 }, /* Prefetch abort */ 258 [4] = { 8, 8 }, /* Data abort */ 259 [5] = { 0, 0 }, /* HVC, unused */ 260 [6] = { 4, 4 }, /* IRQ, unused */ 261 [7] = { 4, 4 }, /* FIQ, unused */ 262 }; 263 264 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) 265 { 266 unsigned long spsr = *vcpu_cpsr(vcpu); 267 bool is_thumb = (spsr & PSR_AA32_T_BIT); 268 u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 269 u32 return_address; 270 271 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); 272 return_address = *vcpu_pc(vcpu); 273 return_address += return_offsets[vect_offset >> 2][is_thumb]; 274 275 /* KVM only enters the ABT and UND modes, so only deal with those */ 276 switch(mode) { 277 case PSR_AA32_MODE_ABT: 278 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); 279 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; 280 break; 281 282 case PSR_AA32_MODE_UND: 283 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); 284 vcpu_gp_regs(vcpu)->compat_lr_und = return_address; 285 break; 286 } 287 288 /* Branch to exception vector */ 289 if (sctlr & (1 << 13)) 290 vect_offset += 0xffff0000; 291 else /* always have security exceptions */ 292 vect_offset += vcpu_read_sys_reg(vcpu, VBAR_EL1); 293 294 *vcpu_pc(vcpu) = vect_offset; 295 } 296 297 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 298 { 299 if (vcpu_el1_is_32bit(vcpu)) { 300 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 301 case unpack_vcpu_flag(EXCEPT_AA32_UND): 302 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4); 303 break; 304 case unpack_vcpu_flag(EXCEPT_AA32_IABT): 305 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12); 306 break; 307 case unpack_vcpu_flag(EXCEPT_AA32_DABT): 308 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16); 309 break; 310 default: 311 /* Err... */ 312 break; 313 } 314 } else { 315 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 316 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC): 317 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); 318 break; 319 320 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR): 321 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_serror); 322 break; 323 324 case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC): 325 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync); 326 break; 327 328 case unpack_vcpu_flag(EXCEPT_AA64_EL2_IRQ): 329 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq); 330 break; 331 332 case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR): 333 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_serror); 334 break; 335 336 default: 337 /* 338 * Only EL1_{SYNC,SERR} and EL2_{SYNC,IRQ,SERR} makes 339 * sense so far. Everything else gets silently 340 * ignored. 341 */ 342 break; 343 } 344 } 345 } 346 347 /* 348 * Adjust the guest PC (and potentially exception state) depending on 349 * flags provided by the emulation code. 350 */ 351 void __kvm_adjust_pc(struct kvm_vcpu *vcpu) 352 { 353 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { 354 kvm_inject_exception(vcpu); 355 vcpu_clear_flag(vcpu, PENDING_EXCEPTION); 356 vcpu_clear_flag(vcpu, EXCEPT_MASK); 357 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { 358 kvm_skip_instr(vcpu); 359 vcpu_clear_flag(vcpu, INCREMENT_PC); 360 } 361 } 362