1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/include/kvm_emulate.h 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #ifndef __ARM64_KVM_EMULATE_H__ 12 #define __ARM64_KVM_EMULATE_H__ 13 14 #include <linux/bitfield.h> 15 #include <linux/kvm_host.h> 16 17 #include <asm/debug-monitors.h> 18 #include <asm/esr.h> 19 #include <asm/kvm_arm.h> 20 #include <asm/kvm_hyp.h> 21 #include <asm/kvm_nested.h> 22 #include <asm/ptrace.h> 23 #include <asm/cputype.h> 24 #include <asm/virt.h> 25 26 #define CURRENT_EL_SP_EL0_VECTOR 0x0 27 #define CURRENT_EL_SP_ELx_VECTOR 0x200 28 #define LOWER_EL_AArch64_VECTOR 0x400 29 #define LOWER_EL_AArch32_VECTOR 0x600 30 31 enum exception_type { 32 except_type_sync = 0, 33 except_type_irq = 0x80, 34 except_type_fiq = 0x100, 35 except_type_serror = 0x180, 36 }; 37 38 #define kvm_exception_type_names \ 39 { except_type_sync, "SYNC" }, \ 40 { except_type_irq, "IRQ" }, \ 41 { except_type_fiq, "FIQ" }, \ 42 { except_type_serror, "SERROR" } 43 44 bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); 45 void kvm_skip_instr32(struct kvm_vcpu *vcpu); 46 47 void kvm_inject_undefined(struct kvm_vcpu *vcpu); 48 int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr); 49 int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr); 50 int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr); 51 void kvm_inject_size_fault(struct kvm_vcpu *vcpu); 52 53 static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr) 54 { 55 return kvm_inject_sea(vcpu, false, addr); 56 } 57 58 static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr) 59 { 60 return kvm_inject_sea(vcpu, true, addr); 61 } 62 63 static inline int kvm_inject_serror(struct kvm_vcpu *vcpu) 64 { 65 /* 66 * ESR_ELx.ISV (later renamed to IDS) indicates whether or not 67 * ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information. 68 * 69 * Set the bit when injecting an SError w/o an ESR to indicate ISS 70 * does not follow the architected format. 71 */ 72 return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV); 73 } 74 75 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); 76 77 void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); 78 int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); 79 int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); 80 int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr); 81 int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr); 82 83 static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu) 84 { 85 u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) | 86 ESR_ELx_IL; 87 88 kvm_inject_nested_sync(vcpu, esr); 89 } 90 91 #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__) 92 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) 93 { 94 return !(vcpu->arch.hcr_el2 & HCR_RW); 95 } 96 #else 97 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) 98 { 99 return vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT); 100 } 101 #endif 102 103 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) 104 { 105 if (!vcpu_has_run_once(vcpu)) 106 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; 107 108 /* 109 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C 110 * get set in SCTLR_EL1 such that we can detect when the guest 111 * MMU gets turned on and do the necessary cache maintenance 112 * then. 113 */ 114 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 115 vcpu->arch.hcr_el2 |= HCR_TVM; 116 } 117 118 static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) 119 { 120 return (unsigned long *)&vcpu->arch.hcr_el2; 121 } 122 123 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu) 124 { 125 return vcpu->arch.vsesr_el2; 126 } 127 128 static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr) 129 { 130 vcpu->arch.vsesr_el2 = vsesr; 131 } 132 133 static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) 134 { 135 return (unsigned long *)&vcpu_gp_regs(vcpu)->pc; 136 } 137 138 static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) 139 { 140 return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate; 141 } 142 143 static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) 144 { 145 return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT); 146 } 147 148 static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) 149 { 150 if (vcpu_mode_is_32bit(vcpu)) 151 return kvm_condition_valid32(vcpu); 152 153 return true; 154 } 155 156 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) 157 { 158 *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT; 159 } 160 161 /* 162 * vcpu_get_reg and vcpu_set_reg should always be passed a register number 163 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on 164 * AArch32 with banked registers. 165 */ 166 static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu, 167 u8 reg_num) 168 { 169 return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num]; 170 } 171 172 static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, 173 unsigned long val) 174 { 175 if (reg_num != 31) 176 vcpu_gp_regs(vcpu)->regs[reg_num] = val; 177 } 178 179 static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) 180 { 181 switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) { 182 case PSR_MODE_EL2h: 183 case PSR_MODE_EL2t: 184 return true; 185 default: 186 return false; 187 } 188 } 189 190 static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu) 191 { 192 return vcpu_is_el2_ctxt(&vcpu->arch.ctxt); 193 } 194 195 static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu) 196 { 197 return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) || 198 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H)); 199 } 200 201 static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu) 202 { 203 return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE; 204 } 205 206 static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) 207 { 208 /* 209 * DDI0487L.b Known Issue D22105 210 * 211 * When executing at EL2 and HCR_EL2.{E2H,TGE} = {1, 0} it is 212 * IMPLEMENTATION DEFINED whether the effective value of HCR_EL2.AMO 213 * is the value programmed or 1. 214 * 215 * Make the implementation choice of treating the effective value as 1 as 216 * we cannot subsequently catch changes to TGE or AMO that would 217 * otherwise lead to the SError becoming deliverable. 218 */ 219 if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu)) 220 return true; 221 222 return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO; 223 } 224 225 static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) 226 { 227 bool e2h, tge; 228 u64 hcr; 229 230 if (!vcpu_has_nv(vcpu)) 231 return false; 232 233 hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 234 235 e2h = (hcr & HCR_E2H); 236 tge = (hcr & HCR_TGE); 237 238 /* 239 * We are in a hypervisor context if the vcpu mode is EL2 or 240 * E2H and TGE bits are set. The latter means we are in the user space 241 * of the VHE kernel. ARMv8.1 ARM describes this as 'InHost' 242 * 243 * Note that the HCR_EL2.{E2H,TGE}={0,1} isn't really handled in the 244 * rest of the KVM code, and will result in a misbehaving guest. 245 */ 246 return vcpu_is_el2(vcpu) || (e2h && tge) || tge; 247 } 248 249 static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu) 250 { 251 return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu); 252 } 253 254 static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu) 255 { 256 return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu); 257 } 258 259 static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu) 260 { 261 if (!is_nested_ctxt(vcpu)) 262 return false; 263 264 return vcpu_el2_amo_is_set(vcpu) || 265 (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); 266 } 267 268 /* 269 * The layout of SPSR for an AArch32 state is different when observed from an 270 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32 271 * view given an AArch64 view. 272 * 273 * In ARM DDI 0487E.a see: 274 * 275 * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426 276 * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256 277 * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280 278 * 279 * Which show the following differences: 280 * 281 * | Bit | AA64 | AA32 | Notes | 282 * +-----+------+------+-----------------------------| 283 * | 24 | DIT | J | J is RES0 in ARMv8 | 284 * | 21 | SS | DIT | SS doesn't exist in AArch32 | 285 * 286 * ... and all other bits are (currently) common. 287 */ 288 static inline unsigned long host_spsr_to_spsr32(unsigned long spsr) 289 { 290 const unsigned long overlap = BIT(24) | BIT(21); 291 unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT); 292 293 spsr &= ~overlap; 294 295 spsr |= dit << 21; 296 297 return spsr; 298 } 299 300 static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) 301 { 302 u32 mode; 303 304 if (vcpu_mode_is_32bit(vcpu)) { 305 mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK; 306 return mode > PSR_AA32_MODE_USR; 307 } 308 309 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK; 310 311 return mode != PSR_MODE_EL0t; 312 } 313 314 static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) 315 { 316 return vcpu->arch.fault.esr_el2; 317 } 318 319 static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu) 320 { 321 u64 esr = kvm_vcpu_get_esr(vcpu); 322 bool is_wfe = !!(esr & ESR_ELx_WFx_ISS_WFE); 323 u64 hcr_el2 = __vcpu_sys_reg(vcpu, HCR_EL2); 324 325 if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) 326 return false; 327 328 return ((is_wfe && (hcr_el2 & HCR_TWE)) || 329 (!is_wfe && (hcr_el2 & HCR_TWI))); 330 } 331 332 static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) 333 { 334 u64 esr = kvm_vcpu_get_esr(vcpu); 335 336 if (esr & ESR_ELx_CV) 337 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; 338 339 return -1; 340 } 341 342 static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu) 343 { 344 return vcpu->arch.fault.far_el2; 345 } 346 347 static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) 348 { 349 u64 hpfar = vcpu->arch.fault.hpfar_el2; 350 351 if (unlikely(!(hpfar & HPFAR_EL2_NS))) 352 return INVALID_GPA; 353 354 return FIELD_GET(HPFAR_EL2_FIPA, hpfar) << 12; 355 } 356 357 static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu) 358 { 359 return vcpu->arch.fault.disr_el1; 360 } 361 362 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) 363 { 364 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK; 365 } 366 367 static __always_inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu) 368 { 369 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_ISV); 370 } 371 372 static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu) 373 { 374 return kvm_vcpu_get_esr(vcpu) & (ESR_ELx_CM | ESR_ELx_WNR | ESR_ELx_FSC); 375 } 376 377 static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu) 378 { 379 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SSE); 380 } 381 382 static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu) 383 { 384 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SF); 385 } 386 387 static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) 388 { 389 return (kvm_vcpu_get_esr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT; 390 } 391 392 static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu) 393 { 394 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW); 395 } 396 397 /* Always check for S1PTW *before* using this. */ 398 static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) 399 { 400 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR; 401 } 402 403 static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu) 404 { 405 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM); 406 } 407 408 static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) 409 { 410 return 1 << ((kvm_vcpu_get_esr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT); 411 } 412 413 /* This one is not specific to Data Abort */ 414 static __always_inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) 415 { 416 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_IL); 417 } 418 419 static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) 420 { 421 return ESR_ELx_EC(kvm_vcpu_get_esr(vcpu)); 422 } 423 424 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) 425 { 426 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; 427 } 428 429 static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu) 430 { 431 return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); 432 } 433 434 static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) 435 { 436 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; 437 } 438 439 static inline 440 bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) 441 { 442 return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(vcpu)); 443 } 444 445 static inline 446 bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) 447 { 448 return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); 449 } 450 451 static inline 452 u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) 453 { 454 unsigned long esr = kvm_vcpu_get_esr(vcpu); 455 456 BUG_ON(!esr_fsc_is_permission_fault(esr)); 457 return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); 458 } 459 460 static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) 461 { 462 switch (kvm_vcpu_trap_get_fault(vcpu)) { 463 case ESR_ELx_FSC_EXTABT: 464 case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): 465 case ESR_ELx_FSC_SECC: 466 case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): 467 return true; 468 default: 469 return false; 470 } 471 } 472 473 static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) 474 { 475 u64 esr = kvm_vcpu_get_esr(vcpu); 476 return ESR_ELx_SYS64_ISS_RT(esr); 477 } 478 479 static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) 480 { 481 if (kvm_vcpu_abt_iss1tw(vcpu)) { 482 /* 483 * Only a permission fault on a S1PTW should be 484 * considered as a write. Otherwise, page tables baked 485 * in a read-only memslot will result in an exception 486 * being delivered in the guest. 487 * 488 * The drawback is that we end-up faulting twice if the 489 * guest is using any of HW AF/DB: a translation fault 490 * to map the page containing the PT (read only at 491 * first), then a permission fault to allow the flags 492 * to be set. 493 */ 494 return kvm_vcpu_trap_is_permission_fault(vcpu); 495 } 496 497 if (kvm_vcpu_trap_is_iabt(vcpu)) 498 return false; 499 500 return kvm_vcpu_dabt_iswrite(vcpu); 501 } 502 503 static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) 504 { 505 return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; 506 } 507 508 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) 509 { 510 if (vcpu_mode_is_32bit(vcpu)) { 511 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT; 512 } else { 513 enum vcpu_sysreg r; 514 u64 sctlr; 515 516 r = vcpu_has_nv(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 517 518 sctlr = vcpu_read_sys_reg(vcpu, r); 519 sctlr |= SCTLR_ELx_EE; 520 vcpu_write_sys_reg(vcpu, sctlr, r); 521 } 522 } 523 524 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) 525 { 526 enum vcpu_sysreg r; 527 u64 bit; 528 529 if (vcpu_mode_is_32bit(vcpu)) 530 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT); 531 532 r = is_hyp_ctxt(vcpu) ? SCTLR_EL2 : SCTLR_EL1; 533 bit = vcpu_mode_priv(vcpu) ? SCTLR_ELx_EE : SCTLR_EL1_E0E; 534 535 return vcpu_read_sys_reg(vcpu, r) & bit; 536 } 537 538 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, 539 unsigned long data, 540 unsigned int len) 541 { 542 if (kvm_vcpu_is_be(vcpu)) { 543 switch (len) { 544 case 1: 545 return data & 0xff; 546 case 2: 547 return be16_to_cpu(data & 0xffff); 548 case 4: 549 return be32_to_cpu(data & 0xffffffff); 550 default: 551 return be64_to_cpu(data); 552 } 553 } else { 554 switch (len) { 555 case 1: 556 return data & 0xff; 557 case 2: 558 return le16_to_cpu(data & 0xffff); 559 case 4: 560 return le32_to_cpu(data & 0xffffffff); 561 default: 562 return le64_to_cpu(data); 563 } 564 } 565 566 return data; /* Leave LE untouched */ 567 } 568 569 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu, 570 unsigned long data, 571 unsigned int len) 572 { 573 if (kvm_vcpu_is_be(vcpu)) { 574 switch (len) { 575 case 1: 576 return data & 0xff; 577 case 2: 578 return cpu_to_be16(data & 0xffff); 579 case 4: 580 return cpu_to_be32(data & 0xffffffff); 581 default: 582 return cpu_to_be64(data); 583 } 584 } else { 585 switch (len) { 586 case 1: 587 return data & 0xff; 588 case 2: 589 return cpu_to_le16(data & 0xffff); 590 case 4: 591 return cpu_to_le32(data & 0xffffffff); 592 default: 593 return cpu_to_le64(data); 594 } 595 } 596 597 return data; /* Leave LE untouched */ 598 } 599 600 static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) 601 { 602 WARN_ON(vcpu_get_flag(vcpu, PENDING_EXCEPTION)); 603 vcpu_set_flag(vcpu, INCREMENT_PC); 604 } 605 606 #define kvm_pend_exception(v, e) \ 607 do { \ 608 WARN_ON(vcpu_get_flag((v), INCREMENT_PC)); \ 609 vcpu_set_flag((v), PENDING_EXCEPTION); \ 610 vcpu_set_flag((v), e); \ 611 } while (0) 612 613 /* 614 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE 615 * format if E2H isn't set. 616 */ 617 static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu) 618 { 619 u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2); 620 621 if (!vcpu_el2_e2h_is_set(vcpu)) 622 cptr = translate_cptr_el2_to_cpacr_el1(cptr); 623 624 return cptr; 625 } 626 627 static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu, 628 unsigned int xen) 629 { 630 switch (xen) { 631 case 0b00: 632 case 0b10: 633 return true; 634 case 0b01: 635 return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu); 636 case 0b11: 637 default: 638 return false; 639 } 640 } 641 642 #define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen) \ 643 (!vcpu_has_nv(vcpu) ? false : \ 644 ____cptr_xen_trap_enabled(vcpu, \ 645 SYS_FIELD_GET(CPACR_EL1, xen, \ 646 vcpu_sanitised_cptr_el2(vcpu)))) 647 648 static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu) 649 { 650 return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN); 651 } 652 653 static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu) 654 { 655 return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN); 656 } 657 658 static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) 659 { 660 struct kvm *kvm = vcpu->kvm; 661 662 if (cpus_have_final_cap(ARM64_HAS_HCX)) { 663 /* 664 * In general, all HCRX_EL2 bits are gated by a feature. 665 * The only reason we can set SMPME without checking any 666 * feature is that its effects are not directly observable 667 * from the guest. 668 */ 669 vcpu->arch.hcrx_el2 = HCRX_EL2_SMPME; 670 671 if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 672 vcpu->arch.hcrx_el2 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 673 674 if (kvm_has_tcr2(kvm)) 675 vcpu->arch.hcrx_el2 |= HCRX_EL2_TCR2En; 676 677 if (kvm_has_fpmr(kvm)) 678 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM; 679 680 if (kvm_has_sctlr2(kvm)) 681 vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En; 682 683 if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 684 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnALS; 685 686 if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 687 vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; 688 } 689 } 690 #endif /* __ARM64_KVM_EMULATE_H__ */ 691