1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/include/kvm_emulate.h 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #ifndef __ARM64_KVM_EMULATE_H__ 12 #define __ARM64_KVM_EMULATE_H__ 13 14 #include <linux/bitfield.h> 15 #include <linux/kvm_host.h> 16 17 #include <asm/debug-monitors.h> 18 #include <asm/esr.h> 19 #include <asm/kvm_arm.h> 20 #include <asm/kvm_hyp.h> 21 #include <asm/kvm_nested.h> 22 #include <asm/ptrace.h> 23 #include <asm/cputype.h> 24 #include <asm/virt.h> 25 26 #define CURRENT_EL_SP_EL0_VECTOR 0x0 27 #define CURRENT_EL_SP_ELx_VECTOR 0x200 28 #define LOWER_EL_AArch64_VECTOR 0x400 29 #define LOWER_EL_AArch32_VECTOR 0x600 30 31 enum exception_type { 32 except_type_sync = 0, 33 except_type_irq = 0x80, 34 except_type_fiq = 0x100, 35 except_type_serror = 0x180, 36 }; 37 38 #define kvm_exception_type_names \ 39 { except_type_sync, "SYNC" }, \ 40 { except_type_irq, "IRQ" }, \ 41 { except_type_fiq, "FIQ" }, \ 42 { except_type_serror, "SERROR" } 43 44 bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); 45 void kvm_skip_instr32(struct kvm_vcpu *vcpu); 46 47 void kvm_inject_undefined(struct kvm_vcpu *vcpu); 48 void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr); 49 int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr); 50 int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr); 51 int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr); 52 void kvm_inject_size_fault(struct kvm_vcpu *vcpu); 53 54 static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr) 55 { 56 return kvm_inject_sea(vcpu, false, addr); 57 } 58 59 static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr) 60 { 61 return kvm_inject_sea(vcpu, true, addr); 62 } 63 64 static inline int kvm_inject_serror(struct kvm_vcpu *vcpu) 65 { 66 /* 67 * ESR_ELx.ISV (later renamed to IDS) indicates whether or not 68 * ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information. 69 * 70 * Set the bit when injecting an SError w/o an ESR to indicate ISS 71 * does not follow the architected format. 72 */ 73 return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV); 74 } 75 76 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); 77 78 void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); 79 int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); 80 int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); 81 int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr); 82 int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr); 83 84 static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu) 85 { 86 u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) | 87 ESR_ELx_IL; 88 89 kvm_inject_nested_sync(vcpu, esr); 90 } 91 92 #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__) 93 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) 94 { 95 return !(vcpu->arch.hcr_el2 & HCR_RW); 96 } 97 #else 98 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) 99 { 100 return vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT); 101 } 102 #endif 103 104 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) 105 { 106 if (!vcpu_has_run_once(vcpu)) 107 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; 108 109 /* 110 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C 111 * get set in SCTLR_EL1 such that we can detect when the guest 112 * MMU gets turned on and do the necessary cache maintenance 113 * then. 114 */ 115 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 116 vcpu->arch.hcr_el2 |= HCR_TVM; 117 } 118 119 static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) 120 { 121 return (unsigned long *)&vcpu->arch.hcr_el2; 122 } 123 124 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu) 125 { 126 return vcpu->arch.vsesr_el2; 127 } 128 129 static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr) 130 { 131 vcpu->arch.vsesr_el2 = vsesr; 132 } 133 134 static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) 135 { 136 return (unsigned long *)&vcpu_gp_regs(vcpu)->pc; 137 } 138 139 static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) 140 { 141 return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate; 142 } 143 144 static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) 145 { 146 return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT); 147 } 148 149 static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) 150 { 151 if (vcpu_mode_is_32bit(vcpu)) 152 return kvm_condition_valid32(vcpu); 153 154 return true; 155 } 156 157 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) 158 { 159 *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT; 160 } 161 162 /* 163 * vcpu_get_reg and vcpu_set_reg should always be passed a register number 164 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on 165 * AArch32 with banked registers. 166 */ 167 static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu, 168 u8 reg_num) 169 { 170 return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num]; 171 } 172 173 static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, 174 unsigned long val) 175 { 176 if (reg_num != 31) 177 vcpu_gp_regs(vcpu)->regs[reg_num] = val; 178 } 179 180 static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) 181 { 182 switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) { 183 case PSR_MODE_EL2h: 184 case PSR_MODE_EL2t: 185 return true; 186 default: 187 return false; 188 } 189 } 190 191 static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu) 192 { 193 return vcpu_is_el2_ctxt(&vcpu->arch.ctxt); 194 } 195 196 static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu) 197 { 198 return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) || 199 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H)); 200 } 201 202 static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu) 203 { 204 return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE; 205 } 206 207 static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) 208 { 209 /* 210 * DDI0487L.b Known Issue D22105 211 * 212 * When executing at EL2 and HCR_EL2.{E2H,TGE} = {1, 0} it is 213 * IMPLEMENTATION DEFINED whether the effective value of HCR_EL2.AMO 214 * is the value programmed or 1. 215 * 216 * Make the implementation choice of treating the effective value as 1 as 217 * we cannot subsequently catch changes to TGE or AMO that would 218 * otherwise lead to the SError becoming deliverable. 219 */ 220 if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu)) 221 return true; 222 223 return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO; 224 } 225 226 static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) 227 { 228 bool e2h, tge; 229 u64 hcr; 230 231 if (!vcpu_has_nv(vcpu)) 232 return false; 233 234 hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 235 236 e2h = (hcr & HCR_E2H); 237 tge = (hcr & HCR_TGE); 238 239 /* 240 * We are in a hypervisor context if the vcpu mode is EL2 or 241 * E2H and TGE bits are set. The latter means we are in the user space 242 * of the VHE kernel. ARMv8.1 ARM describes this as 'InHost' 243 * 244 * Note that the HCR_EL2.{E2H,TGE}={0,1} isn't really handled in the 245 * rest of the KVM code, and will result in a misbehaving guest. 246 */ 247 return vcpu_is_el2(vcpu) || (e2h && tge) || tge; 248 } 249 250 static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu) 251 { 252 return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu); 253 } 254 255 static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu) 256 { 257 return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu); 258 } 259 260 static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu) 261 { 262 if (!is_nested_ctxt(vcpu)) 263 return false; 264 265 return vcpu_el2_amo_is_set(vcpu) || 266 (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); 267 } 268 269 static inline bool kvm_has_nv2(struct kvm *kvm) 270 { 271 return (cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 272 kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)); 273 } 274 275 static inline bool kvm_has_nv3(struct kvm *kvm) 276 { 277 return (cpus_have_final_cap(ARM64_HAS_NV3) && 278 kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)); 279 } 280 281 static inline bool is_nested_nv3_ctxt(struct kvm_vcpu *vcpu) 282 { 283 return (has_vhe() && kvm_has_nv3(vcpu->kvm) && is_nested_ctxt(vcpu) && 284 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && 285 (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)); 286 } 287 288 /* 289 * The layout of SPSR for an AArch32 state is different when observed from an 290 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32 291 * view given an AArch64 view. 292 * 293 * In ARM DDI 0487E.a see: 294 * 295 * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426 296 * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256 297 * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280 298 * 299 * Which show the following differences: 300 * 301 * | Bit | AA64 | AA32 | Notes | 302 * +-----+------+------+-----------------------------| 303 * | 24 | DIT | J | J is RES0 in ARMv8 | 304 * | 21 | SS | DIT | SS doesn't exist in AArch32 | 305 * 306 * ... and all other bits are (currently) common. 307 */ 308 static inline unsigned long host_spsr_to_spsr32(unsigned long spsr) 309 { 310 const unsigned long overlap = BIT(24) | BIT(21); 311 unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT); 312 313 spsr &= ~overlap; 314 315 spsr |= dit << 21; 316 317 return spsr; 318 } 319 320 static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) 321 { 322 u32 mode; 323 324 if (vcpu_mode_is_32bit(vcpu)) { 325 mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK; 326 return mode > PSR_AA32_MODE_USR; 327 } 328 329 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK; 330 331 return mode != PSR_MODE_EL0t; 332 } 333 334 static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) 335 { 336 return vcpu->arch.fault.esr_el2; 337 } 338 339 static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu) 340 { 341 u64 esr = kvm_vcpu_get_esr(vcpu); 342 bool is_wfe = !!(esr & ESR_ELx_WFx_ISS_WFE); 343 u64 hcr_el2 = __vcpu_sys_reg(vcpu, HCR_EL2); 344 345 if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) 346 return false; 347 348 return ((is_wfe && (hcr_el2 & HCR_TWE)) || 349 (!is_wfe && (hcr_el2 & HCR_TWI))); 350 } 351 352 static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) 353 { 354 u64 esr = kvm_vcpu_get_esr(vcpu); 355 356 if (esr & ESR_ELx_CV) 357 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 358 359 return -1; 360 } 361 362 static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu) 363 { 364 return vcpu->arch.fault.far_el2; 365 } 366 367 static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) 368 { 369 u64 hpfar = vcpu->arch.fault.hpfar_el2; 370 371 if (unlikely(!(hpfar & HPFAR_EL2_NS))) 372 return INVALID_GPA; 373 374 return FIELD_GET(HPFAR_EL2_FIPA, hpfar) << 12; 375 } 376 377 static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu) 378 { 379 return vcpu->arch.fault.disr_el1; 380 } 381 382 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) 383 { 384 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK; 385 } 386 387 static __always_inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu) 388 { 389 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_ISV); 390 } 391 392 static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu) 393 { 394 return kvm_vcpu_get_esr(vcpu) & (ESR_ELx_CM | ESR_ELx_WNR | ESR_ELx_FSC); 395 } 396 397 static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu) 398 { 399 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SSE); 400 } 401 402 static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu) 403 { 404 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SF); 405 } 406 407 static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) 408 { 409 return (kvm_vcpu_get_esr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT; 410 } 411 412 static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu) 413 { 414 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW); 415 } 416 417 /* Always check for S1PTW *before* using this. */ 418 static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) 419 { 420 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR; 421 } 422 423 static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu) 424 { 425 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM); 426 } 427 428 static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) 429 { 430 return 1 << ((kvm_vcpu_get_esr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT); 431 } 432 433 /* This one is not specific to Data Abort */ 434 static __always_inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) 435 { 436 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_IL); 437 } 438 439 static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) 440 { 441 return ESR_ELx_EC(kvm_vcpu_get_esr(vcpu)); 442 } 443 444 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) 445 { 446 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; 447 } 448 449 static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu) 450 { 451 return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); 452 } 453 454 static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) 455 { 456 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; 457 } 458 459 static inline 460 bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) 461 { 462 return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(vcpu)); 463 } 464 465 static inline 466 bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) 467 { 468 return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); 469 } 470 471 static inline 472 u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) 473 { 474 unsigned long esr = kvm_vcpu_get_esr(vcpu); 475 476 BUG_ON(!esr_fsc_is_permission_fault(esr)); 477 return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); 478 } 479 480 static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) 481 { 482 switch (kvm_vcpu_trap_get_fault(vcpu)) { 483 case ESR_ELx_FSC_EXTABT: 484 case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): 485 case ESR_ELx_FSC_SECC: 486 case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): 487 return true; 488 default: 489 return false; 490 } 491 } 492 493 static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) 494 { 495 u64 esr = kvm_vcpu_get_esr(vcpu); 496 return ESR_ELx_SYS64_ISS_RT(esr); 497 } 498 499 static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) 500 { 501 if (kvm_vcpu_abt_iss1tw(vcpu)) { 502 /* 503 * Only a permission fault on a S1PTW should be 504 * considered as a write. Otherwise, page tables baked 505 * in a read-only memslot will result in an exception 506 * being delivered in the guest. 507 * 508 * The drawback is that we end-up faulting twice if the 509 * guest is using any of HW AF/DB: a translation fault 510 * to map the page containing the PT (read only at 511 * first), then a permission fault to allow the flags 512 * to be set. 513 */ 514 return kvm_vcpu_trap_is_permission_fault(vcpu); 515 } 516 517 if (kvm_vcpu_trap_is_iabt(vcpu)) 518 return false; 519 520 return kvm_vcpu_dabt_iswrite(vcpu); 521 } 522 523 static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) 524 { 525 return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; 526 } 527 528 /* In nVHE hyp code, registers are always in memory: use the raw accessors. */ 529 #if defined(__KVM_NVHE_HYPERVISOR__) 530 #define vcpu_read_sys_reg(v, r) __vcpu_sys_reg(v, r) 531 #define vcpu_write_sys_reg(v, x, r) __vcpu_assign_sys_reg(v, r, x) 532 #endif 533 534 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) 535 { 536 if (vcpu_mode_is_32bit(vcpu)) { 537 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT; 538 } else { 539 enum vcpu_sysreg r; 540 u64 sctlr; 541 542 r = vcpu_has_nv(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 543 544 sctlr = vcpu_read_sys_reg(vcpu, r); 545 sctlr |= SCTLR_ELx_EE; 546 vcpu_write_sys_reg(vcpu, sctlr, r); 547 } 548 } 549 550 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) 551 { 552 enum vcpu_sysreg r; 553 u64 bit; 554 555 if (vcpu_mode_is_32bit(vcpu)) 556 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT); 557 558 r = is_hyp_ctxt(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 559 bit = vcpu_mode_priv(vcpu) ? SCTLR_ELx_EE : SCTLR_EL1_E0E; 560 561 return vcpu_read_sys_reg(vcpu, r) & bit; 562 } 563 564 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, 565 unsigned long data, 566 unsigned int len) 567 { 568 if (kvm_vcpu_is_be(vcpu)) { 569 switch (len) { 570 case 1: 571 return data & 0xff; 572 case 2: 573 return be16_to_cpu(data & 0xffff); 574 case 4: 575 return be32_to_cpu(data & 0xffffffff); 576 default: 577 return be64_to_cpu(data); 578 } 579 } else { 580 switch (len) { 581 case 1: 582 return data & 0xff; 583 case 2: 584 return le16_to_cpu(data & 0xffff); 585 case 4: 586 return le32_to_cpu(data & 0xffffffff); 587 default: 588 return le64_to_cpu(data); 589 } 590 } 591 592 return data; /* Leave LE untouched */ 593 } 594 595 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu, 596 unsigned long data, 597 unsigned int len) 598 { 599 if (kvm_vcpu_is_be(vcpu)) { 600 switch (len) { 601 case 1: 602 return data & 0xff; 603 case 2: 604 return cpu_to_be16(data & 0xffff); 605 case 4: 606 return cpu_to_be32(data & 0xffffffff); 607 default: 608 return cpu_to_be64(data); 609 } 610 } else { 611 switch (len) { 612 case 1: 613 return data & 0xff; 614 case 2: 615 return cpu_to_le16(data & 0xffff); 616 case 4: 617 return cpu_to_le32(data & 0xffffffff); 618 default: 619 return cpu_to_le64(data); 620 } 621 } 622 623 return data; /* Leave LE untouched */ 624 } 625 626 static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) 627 { 628 WARN_ON(vcpu_get_flag(vcpu, PENDING_EXCEPTION)); 629 vcpu_set_flag(vcpu, INCREMENT_PC); 630 } 631 632 #define kvm_pend_exception(v, e) \ 633 do { \ 634 WARN_ON(vcpu_get_flag((v), INCREMENT_PC)); \ 635 vcpu_set_flag((v), PENDING_EXCEPTION); \ 636 vcpu_set_flag((v), e); \ 637 } while (0) 638 639 /* 640 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE 641 * format if E2H isn't set. 642 */ 643 static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu) 644 { 645 u64 cptr = vcpu_read_sys_reg(vcpu, CPTR_EL2); 646 647 if (!vcpu_el2_e2h_is_set(vcpu)) 648 cptr = translate_cptr_el2_to_cpacr_el1(cptr); 649 650 return cptr; 651 } 652 653 static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu, 654 unsigned int xen) 655 { 656 switch (xen) { 657 case 0b00: 658 case 0b10: 659 return true; 660 case 0b01: 661 return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu); 662 case 0b11: 663 default: 664 return false; 665 } 666 } 667 668 #define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen) \ 669 (!vcpu_has_nv(vcpu) ? false : \ 670 ____cptr_xen_trap_enabled(vcpu, \ 671 SYS_FIELD_GET(CPACR_EL1, xen, \ 672 vcpu_sanitised_cptr_el2(vcpu)))) 673 674 static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu) 675 { 676 return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN); 677 } 678 679 static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu) 680 { 681 return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN); 682 } 683 684 static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) 685 { 686 struct kvm *kvm = vcpu->kvm; 687 688 if (cpus_have_final_cap(ARM64_HAS_HCX)) { 689 /* 690 * In general, all HCRX_EL2 bits are gated by a feature. 691 * The only reason we can set SMPME without checking any 692 * feature is that its effects are not directly observable 693 * from the guest. 694 */ 695 vcpu->arch.hcrx_el2 = HCRX_EL2_SMPME; 696 697 if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 698 vcpu->arch.hcrx_el2 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 699 700 if (kvm_has_tcr2(kvm)) 701 vcpu->arch.hcrx_el2 |= HCRX_EL2_TCR2En; 702 703 if (kvm_has_fpmr(kvm)) 704 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM; 705 706 if (kvm_has_sctlr2(kvm)) 707 vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En; 708 709 if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 710 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnALS; 711 712 if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 713 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; 714 715 /* 716 * NV3 is a host-specific extension, and we always use 717 * it when present and that the guest uses NV. It may 718 * be hidden from the guest though. 719 */ 720 if (cpus_have_final_cap(ARM64_HAS_NV3) && 721 vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) { 722 vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE; 723 724 /* 725 * If the guest is NV2-capable, then we need to see 726 * all the TLBIs, as configured in HCR_EL2. 727 * Otherwise, relax the TLBI traps to only TGE=0. 728 */ 729 if (!kvm_has_nv2(vcpu->kvm)) { 730 vcpu->arch.hcrx_el2 |= (HCRX_EL2_NVnTTLB | 731 HCRX_EL2_NVnTTLBIS); 732 733 if (kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 734 vcpu->arch.hcrx_el2 |= HCRX_EL2_NVnTTLBOS; 735 } 736 } 737 } 738 } 739 740 /* Reset a vcpu's core registers. */ 741 static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) 742 { 743 u32 pstate; 744 745 if (vcpu_el1_is_32bit(vcpu)) 746 pstate = VCPU_RESET_PSTATE_SVC; 747 else if (vcpu_has_nv(vcpu)) 748 pstate = VCPU_RESET_PSTATE_EL2; 749 else 750 pstate = VCPU_RESET_PSTATE_EL1; 751 752 /* Reset core registers */ 753 memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); 754 memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); 755 vcpu->arch.ctxt.spsr_abt = 0; 756 vcpu->arch.ctxt.spsr_und = 0; 757 vcpu->arch.ctxt.spsr_irq = 0; 758 vcpu->arch.ctxt.spsr_fiq = 0; 759 vcpu_gp_regs(vcpu)->pstate = pstate; 760 } 761 762 /* PSCI reset handling for a vcpu. */ 763 static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, 764 struct vcpu_reset_state *reset_state) 765 { 766 unsigned long target_pc = reset_state->pc; 767 768 /* Gracefully handle Thumb2 entry point */ 769 if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { 770 target_pc &= ~1UL; 771 vcpu_set_thumb(vcpu); 772 } 773 774 /* Propagate caller endianness */ 775 if (reset_state->be) 776 kvm_vcpu_set_be(vcpu); 777 778 *vcpu_pc(vcpu) = target_pc; 779 780 /* 781 * We may come from a state where either a PC update was 782 * pending (SMC call resulting in PC being increpented to 783 * skip the SMC) or a pending exception. Make sure we get 784 * rid of all that, as this cannot be valid out of reset. 785 * 786 * Note that clearing the exception mask also clears PC 787 * updates, but that's an implementation detail, and we 788 * really want to make it explicit. 789 */ 790 vcpu_clear_flag(vcpu, PENDING_EXCEPTION); 791 vcpu_clear_flag(vcpu, EXCEPT_MASK); 792 vcpu_clear_flag(vcpu, INCREMENT_PC); 793 vcpu_set_reg(vcpu, 0, reset_state->r0); 794 } 795 796 #endif /* __ARM64_KVM_EMULATE_H__ */ 797