1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #ifndef __ARM64_KVM_HYP_SWITCH_H__ 8 #define __ARM64_KVM_HYP_SWITCH_H__ 9 10 #include <hyp/adjust_pc.h> 11 #include <hyp/fault.h> 12 13 #include <linux/arm-smccc.h> 14 #include <linux/kvm_host.h> 15 #include <linux/types.h> 16 #include <linux/jump_label.h> 17 #include <uapi/linux/psci.h> 18 19 #include <kvm/arm_psci.h> 20 21 #include <asm/barrier.h> 22 #include <asm/cpufeature.h> 23 #include <asm/extable.h> 24 #include <asm/kprobes.h> 25 #include <asm/kvm_asm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/kvm_hyp.h> 28 #include <asm/kvm_mmu.h> 29 #include <asm/kvm_nested.h> 30 #include <asm/fpsimd.h> 31 #include <asm/debug-monitors.h> 32 #include <asm/processor.h> 33 #include <asm/traps.h> 34 35 struct kvm_exception_table_entry { 36 int insn, fixup; 37 }; 38 39 extern struct kvm_exception_table_entry __start___kvm_ex_table; 40 extern struct kvm_exception_table_entry __stop___kvm_ex_table; 41 42 /* Save the 32-bit only FPSIMD system register state */ 43 static inline void __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu) 44 { 45 if (!vcpu_el1_is_32bit(vcpu)) 46 return; 47 48 __vcpu_assign_sys_reg(vcpu, FPEXC32_EL2, read_sysreg(fpexc32_el2)); 49 } 50 51 static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) 52 { 53 /* 54 * We are about to set CPTR_EL2.TFP to trap all floating point 55 * register accesses to EL2, however, the ARM ARM clearly states that 56 * traps are only taken to EL2 if the operation would not otherwise 57 * trap to EL1. Therefore, always make sure that for 32-bit guests, 58 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit. 59 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to 60 * it will cause an exception. 61 */ 62 if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) 63 write_sysreg(1 << 30, fpexc32_el2); 64 } 65 66 static inline void __activate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 67 { 68 u64 val = CPTR_NVHE_EL2_RES1 | CPTR_EL2_TAM | CPTR_EL2_TTA; 69 70 /* 71 * Always trap SME since it's not supported in KVM. 72 * TSM is RES1 if SME isn't implemented. 73 */ 74 val |= CPTR_EL2_TSM; 75 76 if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs()) 77 val |= CPTR_EL2_TZ; 78 79 if (!guest_owns_fp_regs()) 80 val |= CPTR_EL2_TFP; 81 82 write_sysreg(val, cptr_el2); 83 } 84 85 static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 86 { 87 /* 88 * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 89 * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 90 * except for some missing controls, such as TAM. 91 * In this case, CPTR_EL2.TAM has the same position with or without 92 * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 93 * shift value for trapping the AMU accesses. 94 */ 95 u64 val = CPTR_EL2_TAM | CPACR_EL1_TTA; 96 u64 cptr; 97 98 if (guest_owns_fp_regs()) { 99 val |= CPACR_EL1_FPEN; 100 if (vcpu_has_sve(vcpu)) 101 val |= CPACR_EL1_ZEN; 102 } 103 104 if (!vcpu_has_nv(vcpu)) 105 goto write; 106 107 /* 108 * The architecture is a bit crap (what a surprise): an EL2 guest 109 * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, 110 * as they are RES0 in the guest's view. To work around it, trap the 111 * sucker using the very same bit it can't set... 112 */ 113 if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) 114 val |= CPTR_EL2_TCPAC; 115 116 /* 117 * Layer the guest hypervisor's trap configuration on top of our own if 118 * we're in a nested context. 119 */ 120 if (is_hyp_ctxt(vcpu)) 121 goto write; 122 123 cptr = vcpu_sanitised_cptr_el2(vcpu); 124 125 /* 126 * Pay attention, there's some interesting detail here. 127 * 128 * The CPTR_EL2.xEN fields are 2 bits wide, although there are only two 129 * meaningful trap states when HCR_EL2.TGE = 0 (running a nested guest): 130 * 131 * - CPTR_EL2.xEN = x0, traps are enabled 132 * - CPTR_EL2.xEN = x1, traps are disabled 133 * 134 * In other words, bit[0] determines if guest accesses trap or not. In 135 * the interest of simplicity, clear the entire field if the guest 136 * hypervisor has traps enabled to dispel any illusion of something more 137 * complicated taking place. 138 */ 139 if (!(SYS_FIELD_GET(CPACR_EL1, FPEN, cptr) & BIT(0))) 140 val &= ~CPACR_EL1_FPEN; 141 if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0))) 142 val &= ~CPACR_EL1_ZEN; 143 144 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 145 val |= cptr & CPACR_EL1_E0POE; 146 147 val |= cptr & CPTR_EL2_TCPAC; 148 149 write: 150 write_sysreg(val, cpacr_el1); 151 } 152 153 static inline void __activate_cptr_traps(struct kvm_vcpu *vcpu) 154 { 155 if (!guest_owns_fp_regs()) 156 __activate_traps_fpsimd32(vcpu); 157 158 if (has_vhe() || has_hvhe()) 159 __activate_cptr_traps_vhe(vcpu); 160 else 161 __activate_cptr_traps_nvhe(vcpu); 162 } 163 164 static inline void __deactivate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 165 { 166 u64 val = CPTR_NVHE_EL2_RES1; 167 168 if (!cpus_have_final_cap(ARM64_SVE)) 169 val |= CPTR_EL2_TZ; 170 if (!cpus_have_final_cap(ARM64_SME)) 171 val |= CPTR_EL2_TSM; 172 173 write_sysreg(val, cptr_el2); 174 } 175 176 static inline void __deactivate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 177 { 178 u64 val = CPACR_EL1_FPEN; 179 180 if (cpus_have_final_cap(ARM64_SVE)) 181 val |= CPACR_EL1_ZEN; 182 if (cpus_have_final_cap(ARM64_SME)) 183 val |= CPACR_EL1_SMEN; 184 if (cpus_have_final_cap(ARM64_HAS_S1POE)) 185 val |= CPACR_EL1_E0POE; 186 187 write_sysreg(val, cpacr_el1); 188 } 189 190 static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) 191 { 192 if (has_vhe() || has_hvhe()) 193 __deactivate_cptr_traps_vhe(vcpu); 194 else 195 __deactivate_cptr_traps_nvhe(vcpu); 196 } 197 198 static inline bool cpu_has_amu(void) 199 { 200 u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); 201 202 return cpuid_feature_extract_unsigned_field(pfr0, 203 ID_AA64PFR0_EL1_AMU_SHIFT); 204 } 205 206 #define __activate_fgt(hctxt, vcpu, reg) \ 207 do { \ 208 ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \ 209 write_sysreg_s(*vcpu_fgt(vcpu, reg), SYS_ ## reg); \ 210 } while (0) 211 212 static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) 213 { 214 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 215 216 if (!cpus_have_final_cap(ARM64_HAS_FGT)) 217 return; 218 219 __activate_fgt(hctxt, vcpu, HFGRTR_EL2); 220 __activate_fgt(hctxt, vcpu, HFGWTR_EL2); 221 __activate_fgt(hctxt, vcpu, HFGITR_EL2); 222 __activate_fgt(hctxt, vcpu, HDFGRTR_EL2); 223 __activate_fgt(hctxt, vcpu, HDFGWTR_EL2); 224 225 if (cpu_has_amu()) 226 __activate_fgt(hctxt, vcpu, HAFGRTR_EL2); 227 228 if (!cpus_have_final_cap(ARM64_HAS_FGT2)) 229 return; 230 231 __activate_fgt(hctxt, vcpu, HFGRTR2_EL2); 232 __activate_fgt(hctxt, vcpu, HFGWTR2_EL2); 233 __activate_fgt(hctxt, vcpu, HFGITR2_EL2); 234 __activate_fgt(hctxt, vcpu, HDFGRTR2_EL2); 235 __activate_fgt(hctxt, vcpu, HDFGWTR2_EL2); 236 } 237 238 static inline void __activate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu) 239 { 240 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 241 242 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 243 return; 244 245 __activate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2); 246 __activate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2); 247 __activate_fgt(hctxt, vcpu, ICH_HFGITR_EL2); 248 } 249 250 #define __deactivate_fgt(hctxt, vcpu, reg) \ 251 do { \ 252 write_sysreg_s(ctxt_sys_reg(hctxt, reg), \ 253 SYS_ ## reg); \ 254 } while(0) 255 256 static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) 257 { 258 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 259 260 if (!cpus_have_final_cap(ARM64_HAS_FGT)) 261 return; 262 263 __deactivate_fgt(hctxt, vcpu, HFGRTR_EL2); 264 __deactivate_fgt(hctxt, vcpu, HFGWTR_EL2); 265 __deactivate_fgt(hctxt, vcpu, HFGITR_EL2); 266 __deactivate_fgt(hctxt, vcpu, HDFGRTR_EL2); 267 __deactivate_fgt(hctxt, vcpu, HDFGWTR_EL2); 268 269 if (cpu_has_amu()) 270 __deactivate_fgt(hctxt, vcpu, HAFGRTR_EL2); 271 272 if (!cpus_have_final_cap(ARM64_HAS_FGT2)) 273 return; 274 275 __deactivate_fgt(hctxt, vcpu, HFGRTR2_EL2); 276 __deactivate_fgt(hctxt, vcpu, HFGWTR2_EL2); 277 __deactivate_fgt(hctxt, vcpu, HFGITR2_EL2); 278 __deactivate_fgt(hctxt, vcpu, HDFGRTR2_EL2); 279 __deactivate_fgt(hctxt, vcpu, HDFGWTR2_EL2); 280 } 281 282 static inline void __deactivate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu) 283 { 284 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 285 286 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 287 return; 288 289 __deactivate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2); 290 __deactivate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2); 291 __deactivate_fgt(hctxt, vcpu, ICH_HFGITR_EL2); 292 293 } 294 295 static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) 296 { 297 u64 clr = MPAM2_EL2_EnMPAMSM; 298 u64 set = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; 299 300 if (!system_supports_mpam()) 301 return; 302 303 /* trap guest access to MPAMIDR_EL1 */ 304 if (system_supports_mpam_hcr()) { 305 write_sysreg_s(MPAMHCR_EL2_TRAP_MPAMIDR_EL1, SYS_MPAMHCR_EL2); 306 } else { 307 /* From v1.1 TIDR can trap MPAMIDR, set it unconditionally */ 308 set |= MPAM2_EL2_TIDR; 309 } 310 311 sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); 312 } 313 314 static inline void __deactivate_traps_mpam(void) 315 { 316 u64 clr = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1 | MPAM2_EL2_TIDR; 317 u64 set = MPAM2_EL2_EnMPAMSM; 318 319 if (!system_supports_mpam()) 320 return; 321 322 sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); 323 324 if (system_supports_mpam_hcr()) 325 write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); 326 } 327 328 static inline void __activate_traps_common(struct kvm_vcpu *vcpu) 329 { 330 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 331 332 /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */ 333 write_sysreg(1 << 15, hstr_el2); 334 335 /* 336 * Make sure we trap PMU access from EL0 to EL2. Also sanitize 337 * PMSELR_EL0 to make sure it never contains the cycle 338 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at 339 * EL1 instead of being trapped to EL2. 340 */ 341 if (system_supports_pmuv3()) { 342 write_sysreg(0, pmselr_el0); 343 344 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = read_sysreg(pmuserenr_el0); 345 write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0); 346 vcpu_set_flag(vcpu, PMUSERENR_ON_CPU); 347 } 348 349 if (cpus_have_final_cap(ARM64_HAS_HCX)) { 350 u64 hcrx = vcpu->arch.hcrx_el2; 351 if (is_nested_ctxt(vcpu)) { 352 u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2); 353 hcrx |= val & __HCRX_EL2_MASK; 354 hcrx &= ~(~val & __HCRX_EL2_nMASK); 355 } 356 357 ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2); 358 write_sysreg_s(hcrx, SYS_HCRX_EL2); 359 } 360 361 __activate_traps_hfgxtr(vcpu); 362 __activate_traps_ich_hfgxtr(vcpu); 363 __activate_traps_mpam(vcpu); 364 } 365 366 static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) 367 { 368 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 369 370 write_sysreg(0, hstr_el2); 371 if (system_supports_pmuv3()) { 372 write_sysreg(ctxt_sys_reg(hctxt, PMUSERENR_EL0), pmuserenr_el0); 373 vcpu_clear_flag(vcpu, PMUSERENR_ON_CPU); 374 } 375 376 if (cpus_have_final_cap(ARM64_HAS_HCX)) 377 write_sysreg_s(ctxt_sys_reg(hctxt, HCRX_EL2), SYS_HCRX_EL2); 378 379 __deactivate_traps_hfgxtr(vcpu); 380 __deactivate_traps_ich_hfgxtr(vcpu); 381 __deactivate_traps_mpam(); 382 } 383 384 static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr) 385 { 386 if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) 387 hcr |= HCR_TVM; 388 389 write_sysreg_hcr(hcr); 390 391 if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) { 392 u64 vsesr; 393 394 /* 395 * When HCR_EL2.AMO is set, physical SErrors are taken to EL2 396 * and vSError injection is enabled for EL1. Conveniently, for 397 * NV this means that it is never the case where a 'physical' 398 * SError (injected by KVM or userspace) and vSError are 399 * deliverable to the same context. 400 * 401 * As such, we can trivially select between the host or guest's 402 * VSESR_EL2. Except for the case that FEAT_RAS hasn't been 403 * exposed to the guest, where ESR propagation in hardware 404 * occurs unconditionally. 405 * 406 * Paper over the architectural wart and use an IMPLEMENTATION 407 * DEFINED ESR value in case FEAT_RAS is hidden from the guest. 408 */ 409 if (!vserror_state_is_nested(vcpu)) 410 vsesr = vcpu->arch.vsesr_el2; 411 else if (kvm_has_ras(kern_hyp_va(vcpu->kvm))) 412 vsesr = __vcpu_sys_reg(vcpu, VSESR_EL2); 413 else 414 vsesr = ESR_ELx_ISV; 415 416 write_sysreg_s(vsesr, SYS_VSESR_EL2); 417 } 418 } 419 420 static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) 421 { 422 u64 *hcr; 423 424 if (vserror_state_is_nested(vcpu)) 425 hcr = __ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2); 426 else 427 hcr = &vcpu->arch.hcr_el2; 428 429 /* 430 * If we pended a virtual abort, preserve it until it gets 431 * cleared. See D1.14.3 (Virtual Interrupts) for details, but 432 * the crucial bit is "On taking a vSError interrupt, 433 * HCR_EL2.VSE is cleared to 0." 434 * 435 * Additionally, when in a nested context we need to propagate the 436 * updated state to the guest hypervisor's HCR_EL2. 437 */ 438 if (*hcr & HCR_VSE) { 439 *hcr &= ~HCR_VSE; 440 *hcr |= read_sysreg(hcr_el2) & HCR_VSE; 441 } 442 } 443 444 static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) 445 { 446 return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault); 447 } 448 449 static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) 450 { 451 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); 452 arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); 453 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); 454 455 /* 456 * Finish potential single step before executing the prologue 457 * instruction. 458 */ 459 *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; 460 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 461 462 return true; 463 } 464 465 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) 466 { 467 u64 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 468 469 /* 470 * The vCPU's saved SVE state layout always matches the max VL of the 471 * vCPU. Start off with the max VL so we can load the SVE state. 472 */ 473 sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); 474 __sve_restore_state(vcpu_sve_pffr(vcpu), 475 &vcpu->arch.ctxt.fp_regs.fpsr, 476 true); 477 478 /* 479 * The effective VL for a VM could differ from the max VL when running a 480 * nested guest, as the guest hypervisor could select a smaller VL. Slap 481 * that into hardware before wrapping up. 482 */ 483 if (is_nested_ctxt(vcpu)) { 484 zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2)); 485 sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); 486 } 487 488 write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR); 489 } 490 491 static inline void __hyp_sve_save_host(void) 492 { 493 struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); 494 495 sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); 496 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 497 __sve_save_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), 498 &sve_state->fpsr, 499 true); 500 } 501 502 static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) 503 { 504 u64 zcr_el1, zcr_el2; 505 506 if (!guest_owns_fp_regs()) 507 return; 508 509 if (vcpu_has_sve(vcpu)) { 510 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 511 512 /* A guest hypervisor may restrict the effective max VL. */ 513 if (is_nested_ctxt(vcpu)) 514 zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2)); 515 516 write_sysreg_el2(zcr_el2, SYS_ZCR); 517 518 zcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)); 519 write_sysreg_el1(zcr_el1, SYS_ZCR); 520 } 521 } 522 523 static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) 524 { 525 u64 zcr_el1, zcr_el2; 526 527 if (!guest_owns_fp_regs()) 528 return; 529 530 /* 531 * When the guest owns the FP regs, we know that guest+hyp traps for 532 * any FPSIMD/SVE/SME features exposed to the guest have been disabled 533 * by either __activate_cptr_traps() or kvm_hyp_handle_fpsimd() 534 * prior to __guest_entry(). As __guest_entry() guarantees a context 535 * synchronization event, we don't need an ISB here to avoid taking 536 * traps for anything that was exposed to the guest. 537 */ 538 if (vcpu_has_sve(vcpu)) { 539 zcr_el1 = read_sysreg_el1(SYS_ZCR); 540 __vcpu_assign_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu), zcr_el1); 541 542 /* 543 * The guest's state is always saved using the guest's max VL. 544 * Ensure that the host has the guest's max VL active such that 545 * the host can save the guest's state lazily, but don't 546 * artificially restrict the host to the guest's max VL. 547 */ 548 if (has_vhe()) { 549 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 550 write_sysreg_el2(zcr_el2, SYS_ZCR); 551 } else { 552 zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; 553 write_sysreg_el2(zcr_el2, SYS_ZCR); 554 555 zcr_el1 = vcpu_sve_max_vq(vcpu) - 1; 556 write_sysreg_el1(zcr_el1, SYS_ZCR); 557 } 558 } 559 } 560 561 static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) 562 { 563 /* 564 * Non-protected kvm relies on the host restoring its sve state. 565 * Protected kvm restores the host's sve state as not to reveal that 566 * fpsimd was used by a guest nor leak upper sve bits. 567 */ 568 if (system_supports_sve()) { 569 __hyp_sve_save_host(); 570 } else { 571 __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); 572 } 573 574 if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) 575 *host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR); 576 } 577 578 579 /* 580 * We trap the first access to the FP/SIMD to save the host context and 581 * restore the guest context lazily. 582 * If FP/SIMD is not implemented, handle the trap and inject an undefined 583 * instruction exception to the guest. Similarly for trapped SVE accesses. 584 */ 585 static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) 586 { 587 bool sve_guest; 588 u8 esr_ec; 589 590 if (!system_supports_fpsimd()) 591 return false; 592 593 sve_guest = vcpu_has_sve(vcpu); 594 esr_ec = kvm_vcpu_trap_get_class(vcpu); 595 596 /* Only handle traps the vCPU can support here: */ 597 switch (esr_ec) { 598 case ESR_ELx_EC_FP_ASIMD: 599 /* Forward traps to the guest hypervisor as required */ 600 if (guest_hyp_fpsimd_traps_enabled(vcpu)) 601 return false; 602 break; 603 case ESR_ELx_EC_SYS64: 604 if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) 605 return false; 606 fallthrough; 607 case ESR_ELx_EC_SVE: 608 if (!sve_guest) 609 return false; 610 if (guest_hyp_sve_traps_enabled(vcpu)) 611 return false; 612 break; 613 default: 614 return false; 615 } 616 617 /* Valid trap. Switch the context: */ 618 619 /* First disable enough traps to allow us to update the registers */ 620 __deactivate_cptr_traps(vcpu); 621 isb(); 622 623 /* Write out the host state if it's in the registers */ 624 if (is_protected_kvm_enabled() && host_owns_fp_regs()) 625 kvm_hyp_save_fpsimd_host(vcpu); 626 627 /* Restore the guest state */ 628 if (sve_guest) 629 __hyp_sve_restore_guest(vcpu); 630 else 631 __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs); 632 633 if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) 634 write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); 635 636 /* Skip restoring fpexc32 for AArch64 guests */ 637 if (!(read_sysreg(hcr_el2) & HCR_RW)) 638 write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); 639 640 *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; 641 642 /* 643 * Re-enable traps necessary for the current state of the guest, e.g. 644 * those enabled by a guest hypervisor. The ERET to the guest will 645 * provide the necessary context synchronization. 646 */ 647 __activate_cptr_traps(vcpu); 648 649 return true; 650 } 651 652 static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) 653 { 654 u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 655 int rt = kvm_vcpu_sys_get_rt(vcpu); 656 u64 val = vcpu_get_reg(vcpu, rt); 657 658 /* 659 * The normal sysreg handling code expects to see the traps, 660 * let's not do anything here. 661 */ 662 if (vcpu->arch.hcr_el2 & HCR_TVM) 663 return false; 664 665 switch (sysreg) { 666 case SYS_SCTLR_EL1: 667 write_sysreg_el1(val, SYS_SCTLR); 668 break; 669 case SYS_TTBR0_EL1: 670 write_sysreg_el1(val, SYS_TTBR0); 671 break; 672 case SYS_TTBR1_EL1: 673 write_sysreg_el1(val, SYS_TTBR1); 674 break; 675 case SYS_TCR_EL1: 676 write_sysreg_el1(val, SYS_TCR); 677 break; 678 case SYS_ESR_EL1: 679 write_sysreg_el1(val, SYS_ESR); 680 break; 681 case SYS_FAR_EL1: 682 write_sysreg_el1(val, SYS_FAR); 683 break; 684 case SYS_AFSR0_EL1: 685 write_sysreg_el1(val, SYS_AFSR0); 686 break; 687 case SYS_AFSR1_EL1: 688 write_sysreg_el1(val, SYS_AFSR1); 689 break; 690 case SYS_MAIR_EL1: 691 write_sysreg_el1(val, SYS_MAIR); 692 break; 693 case SYS_AMAIR_EL1: 694 write_sysreg_el1(val, SYS_AMAIR); 695 break; 696 case SYS_CONTEXTIDR_EL1: 697 write_sysreg_el1(val, SYS_CONTEXTIDR); 698 break; 699 default: 700 return false; 701 } 702 703 __kvm_skip_instr(vcpu); 704 return true; 705 } 706 707 /* Open-coded version of timer_get_offset() to allow for kern_hyp_va() */ 708 static inline u64 hyp_timer_get_offset(struct arch_timer_context *ctxt) 709 { 710 u64 offset = 0; 711 712 if (ctxt->offset.vm_offset) 713 offset += *kern_hyp_va(ctxt->offset.vm_offset); 714 if (ctxt->offset.vcpu_offset) 715 offset += *kern_hyp_va(ctxt->offset.vcpu_offset); 716 717 return offset; 718 } 719 720 static inline u64 compute_counter_value(struct arch_timer_context *ctxt) 721 { 722 return arch_timer_read_cntpct_el0() - hyp_timer_get_offset(ctxt); 723 } 724 725 static bool kvm_handle_cntxct(struct kvm_vcpu *vcpu) 726 { 727 struct arch_timer_context *ctxt; 728 u32 sysreg; 729 u64 val; 730 731 /* 732 * We only get here for 64bit guests, 32bit guests will hit 733 * the long and winding road all the way to the standard 734 * handling. Yes, it sucks to be irrelevant. 735 * 736 * Also, we only deal with non-hypervisor context here (either 737 * an EL1 guest, or a non-HYP context of an EL2 guest). 738 */ 739 if (is_hyp_ctxt(vcpu)) 740 return false; 741 742 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 743 744 switch (sysreg) { 745 case SYS_CNTPCT_EL0: 746 case SYS_CNTPCTSS_EL0: 747 if (vcpu_has_nv(vcpu)) { 748 /* Check for guest hypervisor trapping */ 749 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2); 750 if (!vcpu_el2_e2h_is_set(vcpu)) 751 val = (val & CNTHCTL_EL1PCTEN) << 10; 752 753 if (!(val & (CNTHCTL_EL1PCTEN << 10))) 754 return false; 755 } 756 757 ctxt = vcpu_ptimer(vcpu); 758 break; 759 case SYS_CNTVCT_EL0: 760 case SYS_CNTVCTSS_EL0: 761 if (vcpu_has_nv(vcpu)) { 762 /* Check for guest hypervisor trapping */ 763 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2); 764 765 if (val & CNTHCTL_EL1TVCT) 766 return false; 767 } 768 769 ctxt = vcpu_vtimer(vcpu); 770 break; 771 default: 772 return false; 773 } 774 775 val = compute_counter_value(ctxt); 776 777 vcpu_set_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu), val); 778 __kvm_skip_instr(vcpu); 779 return true; 780 } 781 782 static bool handle_ampere1_tcr(struct kvm_vcpu *vcpu) 783 { 784 u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 785 int rt = kvm_vcpu_sys_get_rt(vcpu); 786 u64 val = vcpu_get_reg(vcpu, rt); 787 788 if (sysreg != SYS_TCR_EL1) 789 return false; 790 791 /* 792 * Affected parts do not advertise support for hardware Access Flag / 793 * Dirty state management in ID_AA64MMFR1_EL1.HAFDBS, but the underlying 794 * control bits are still functional. The architecture requires these be 795 * RES0 on systems that do not implement FEAT_HAFDBS. 796 * 797 * Uphold the requirements of the architecture by masking guest writes 798 * to TCR_EL1.{HA,HD} here. 799 */ 800 val &= ~(TCR_HD | TCR_HA); 801 write_sysreg_el1(val, SYS_TCR); 802 __kvm_skip_instr(vcpu); 803 return true; 804 } 805 806 static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) 807 { 808 if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && 809 handle_tx2_tvm(vcpu)) 810 return true; 811 812 if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38) && 813 handle_ampere1_tcr(vcpu)) 814 return true; 815 816 if (static_branch_unlikely(&vgic_v3_cpuif_trap) && 817 __vgic_v3_perform_cpuif_access(vcpu) == 1) 818 return true; 819 820 if (kvm_handle_cntxct(vcpu)) 821 return true; 822 823 return false; 824 } 825 826 static inline bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) 827 { 828 if (static_branch_unlikely(&vgic_v3_cpuif_trap) && 829 __vgic_v3_perform_cpuif_access(vcpu) == 1) 830 return true; 831 832 return false; 833 } 834 835 static inline bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu, 836 u64 *exit_code) 837 { 838 if (!__populate_fault_info(vcpu)) 839 return true; 840 841 return false; 842 } 843 #define kvm_hyp_handle_iabt_low kvm_hyp_handle_memory_fault 844 #define kvm_hyp_handle_watchpt_low kvm_hyp_handle_memory_fault 845 846 static inline bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) 847 { 848 if (kvm_hyp_handle_memory_fault(vcpu, exit_code)) 849 return true; 850 851 if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { 852 bool valid; 853 854 valid = kvm_vcpu_trap_is_translation_fault(vcpu) && 855 kvm_vcpu_dabt_isvalid(vcpu) && 856 !kvm_vcpu_abt_issea(vcpu) && 857 !kvm_vcpu_abt_iss1tw(vcpu); 858 859 if (valid) { 860 int ret = __vgic_v2_perform_cpuif_access(vcpu); 861 862 if (ret == 1) 863 return true; 864 865 /* Promote an illegal access to an SError.*/ 866 if (ret == -1) 867 *exit_code = ARM_EXCEPTION_EL1_SERROR; 868 } 869 } 870 871 return false; 872 } 873 874 typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *); 875 876 /* 877 * Allow the hypervisor to handle the exit with an exit handler if it has one. 878 * 879 * Returns true if the hypervisor handled the exit, and control should go back 880 * to the guest, or false if it hasn't. 881 */ 882 static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code, 883 const exit_handler_fn *handlers) 884 { 885 exit_handler_fn fn = handlers[kvm_vcpu_trap_get_class(vcpu)]; 886 if (fn) 887 return fn(vcpu, exit_code); 888 889 return false; 890 } 891 892 static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu) 893 { 894 /* 895 * Check for the conditions of Cortex-A510's #2077057. When these occur 896 * SPSR_EL2 can't be trusted, but isn't needed either as it is 897 * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. 898 * Are we single-stepping the guest, and took a PAC exception from the 899 * active-not-pending state? 900 */ 901 if (cpus_have_final_cap(ARM64_WORKAROUND_2077057) && 902 vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 903 *vcpu_cpsr(vcpu) & DBG_SPSR_SS && 904 ESR_ELx_EC(read_sysreg_el2(SYS_ESR)) == ESR_ELx_EC_PAC) 905 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 906 907 vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR); 908 } 909 910 /* 911 * Return true when we were able to fixup the guest exit and should return to 912 * the guest, false when we should restore the host state and return to the 913 * main run loop. 914 */ 915 static inline bool __fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code, 916 const exit_handler_fn *handlers) 917 { 918 if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) 919 vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); 920 921 if (ARM_SERROR_PENDING(*exit_code) && 922 ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) { 923 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu); 924 925 /* 926 * HVC already have an adjusted PC, which we need to 927 * correct in order to return to after having injected 928 * the SError. 929 * 930 * SMC, on the other hand, is *trapped*, meaning its 931 * preferred return address is the SMC itself. 932 */ 933 if (esr_ec == ESR_ELx_EC_HVC32 || esr_ec == ESR_ELx_EC_HVC64) 934 write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR); 935 } 936 937 /* 938 * We're using the raw exception code in order to only process 939 * the trap if no SError is pending. We will come back to the 940 * same PC once the SError has been injected, and replay the 941 * trapping instruction. 942 */ 943 if (*exit_code != ARM_EXCEPTION_TRAP) 944 goto exit; 945 946 /* Check if there's an exit handler and allow it to handle the exit. */ 947 if (kvm_hyp_handle_exit(vcpu, exit_code, handlers)) 948 goto guest; 949 exit: 950 /* Return to the host kernel and handle the exit */ 951 return false; 952 953 guest: 954 /* Re-enter the guest */ 955 asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); 956 return true; 957 } 958 959 static inline void __kvm_unexpected_el2_exception(void) 960 { 961 extern char __guest_exit_restore_elr_and_panic[]; 962 unsigned long addr, fixup; 963 struct kvm_exception_table_entry *entry, *end; 964 unsigned long elr_el2 = read_sysreg(elr_el2); 965 966 entry = &__start___kvm_ex_table; 967 end = &__stop___kvm_ex_table; 968 969 while (entry < end) { 970 addr = (unsigned long)&entry->insn + entry->insn; 971 fixup = (unsigned long)&entry->fixup + entry->fixup; 972 973 if (addr != elr_el2) { 974 entry++; 975 continue; 976 } 977 978 write_sysreg(fixup, elr_el2); 979 return; 980 } 981 982 /* Trigger a panic after restoring the hyp context. */ 983 this_cpu_ptr(&kvm_hyp_ctxt)->sys_regs[ELR_EL2] = elr_el2; 984 write_sysreg(__guest_exit_restore_elr_and_panic, elr_el2); 985 } 986 987 #endif /* __ARM64_KVM_HYP_SWITCH_H__ */ 988