1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #ifndef __ARM64_KVM_HYP_SWITCH_H__ 8 #define __ARM64_KVM_HYP_SWITCH_H__ 9 10 #include <hyp/adjust_pc.h> 11 #include <hyp/fault.h> 12 13 #include <linux/arm-smccc.h> 14 #include <linux/kvm_host.h> 15 #include <linux/types.h> 16 #include <linux/jump_label.h> 17 #include <uapi/linux/psci.h> 18 19 #include <kvm/arm_psci.h> 20 21 #include <asm/barrier.h> 22 #include <asm/cpufeature.h> 23 #include <asm/extable.h> 24 #include <asm/kprobes.h> 25 #include <asm/kvm_asm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/kvm_hyp.h> 28 #include <asm/kvm_mmu.h> 29 #include <asm/kvm_nested.h> 30 #include <asm/fpsimd.h> 31 #include <asm/debug-monitors.h> 32 #include <asm/processor.h> 33 #include <asm/traps.h> 34 35 struct kvm_exception_table_entry { 36 int insn, fixup; 37 }; 38 39 extern struct kvm_exception_table_entry __start___kvm_ex_table; 40 extern struct kvm_exception_table_entry __stop___kvm_ex_table; 41 42 /* Save the 32-bit only FPSIMD system register state */ 43 static inline void __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu) 44 { 45 if (!vcpu_el1_is_32bit(vcpu)) 46 return; 47 48 __vcpu_assign_sys_reg(vcpu, FPEXC32_EL2, read_sysreg(fpexc32_el2)); 49 } 50 51 static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) 52 { 53 /* 54 * We are about to set CPTR_EL2.TFP to trap all floating point 55 * register accesses to EL2, however, the ARM ARM clearly states that 56 * traps are only taken to EL2 if the operation would not otherwise 57 * trap to EL1. Therefore, always make sure that for 32-bit guests, 58 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit. 59 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to 60 * it will cause an exception. 61 */ 62 if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) 63 write_sysreg(1 << 30, fpexc32_el2); 64 } 65 66 static inline void __activate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 67 { 68 u64 val = CPTR_NVHE_EL2_RES1 | CPTR_EL2_TAM | CPTR_EL2_TTA; 69 70 /* 71 * Always trap SME since it's not supported in KVM. 72 * TSM is RES1 if SME isn't implemented. 73 */ 74 val |= CPTR_EL2_TSM; 75 76 if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs()) 77 val |= CPTR_EL2_TZ; 78 79 if (!guest_owns_fp_regs()) 80 val |= CPTR_EL2_TFP; 81 82 write_sysreg(val, cptr_el2); 83 } 84 85 static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 86 { 87 /* 88 * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 89 * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 90 * except for some missing controls, such as TAM. 91 * In this case, CPTR_EL2.TAM has the same position with or without 92 * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 93 * shift value for trapping the AMU accesses. 94 */ 95 u64 val = CPTR_EL2_TAM | CPACR_EL1_TTA; 96 u64 cptr; 97 98 if (guest_owns_fp_regs()) { 99 val |= CPACR_EL1_FPEN; 100 if (vcpu_has_sve(vcpu)) 101 val |= CPACR_EL1_ZEN; 102 } 103 104 if (!vcpu_has_nv(vcpu)) 105 goto write; 106 107 /* 108 * The architecture is a bit crap (what a surprise): an EL2 guest 109 * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, 110 * as they are RES0 in the guest's view. To work around it, trap the 111 * sucker using the very same bit it can't set... 112 */ 113 if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) 114 val |= CPTR_EL2_TCPAC; 115 116 /* 117 * Layer the guest hypervisor's trap configuration on top of our own if 118 * we're in a nested context. 119 */ 120 if (is_hyp_ctxt(vcpu)) 121 goto write; 122 123 cptr = vcpu_sanitised_cptr_el2(vcpu); 124 125 /* 126 * Pay attention, there's some interesting detail here. 127 * 128 * The CPTR_EL2.xEN fields are 2 bits wide, although there are only two 129 * meaningful trap states when HCR_EL2.TGE = 0 (running a nested guest): 130 * 131 * - CPTR_EL2.xEN = x0, traps are enabled 132 * - CPTR_EL2.xEN = x1, traps are disabled 133 * 134 * In other words, bit[0] determines if guest accesses trap or not. In 135 * the interest of simplicity, clear the entire field if the guest 136 * hypervisor has traps enabled to dispel any illusion of something more 137 * complicated taking place. 138 */ 139 if (!(SYS_FIELD_GET(CPACR_EL1, FPEN, cptr) & BIT(0))) 140 val &= ~CPACR_EL1_FPEN; 141 if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0))) 142 val &= ~CPACR_EL1_ZEN; 143 144 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 145 val |= cptr & CPACR_EL1_E0POE; 146 147 val |= cptr & CPTR_EL2_TCPAC; 148 149 write: 150 write_sysreg(val, cpacr_el1); 151 } 152 153 static inline void __activate_cptr_traps(struct kvm_vcpu *vcpu) 154 { 155 if (!guest_owns_fp_regs()) 156 __activate_traps_fpsimd32(vcpu); 157 158 if (has_vhe() || has_hvhe()) 159 __activate_cptr_traps_vhe(vcpu); 160 else 161 __activate_cptr_traps_nvhe(vcpu); 162 } 163 164 static inline void __deactivate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 165 { 166 u64 val = CPTR_NVHE_EL2_RES1; 167 168 if (!cpus_have_final_cap(ARM64_SVE)) 169 val |= CPTR_EL2_TZ; 170 if (!cpus_have_final_cap(ARM64_SME)) 171 val |= CPTR_EL2_TSM; 172 173 write_sysreg(val, cptr_el2); 174 } 175 176 static inline void __deactivate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 177 { 178 u64 val = CPACR_EL1_FPEN; 179 180 if (cpus_have_final_cap(ARM64_SVE)) 181 val |= CPACR_EL1_ZEN; 182 if (cpus_have_final_cap(ARM64_SME)) 183 val |= CPACR_EL1_SMEN; 184 if (cpus_have_final_cap(ARM64_HAS_S1POE)) 185 val |= CPACR_EL1_E0POE; 186 187 write_sysreg(val, cpacr_el1); 188 } 189 190 static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) 191 { 192 if (has_vhe() || has_hvhe()) 193 __deactivate_cptr_traps_vhe(vcpu); 194 else 195 __deactivate_cptr_traps_nvhe(vcpu); 196 } 197 198 static inline bool cpu_has_amu(void) 199 { 200 u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); 201 202 return cpuid_feature_extract_unsigned_field(pfr0, 203 ID_AA64PFR0_EL1_AMU_SHIFT); 204 } 205 206 #define __activate_fgt(hctxt, vcpu, reg) \ 207 do { \ 208 ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \ 209 write_sysreg_s(*vcpu_fgt(vcpu, reg), SYS_ ## reg); \ 210 } while (0) 211 212 static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu) 213 { 214 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 215 216 if (!cpus_have_final_cap(ARM64_HAS_FGT)) 217 return; 218 219 __activate_fgt(hctxt, vcpu, HFGRTR_EL2); 220 __activate_fgt(hctxt, vcpu, HFGWTR_EL2); 221 __activate_fgt(hctxt, vcpu, HFGITR_EL2); 222 __activate_fgt(hctxt, vcpu, HDFGRTR_EL2); 223 __activate_fgt(hctxt, vcpu, HDFGWTR_EL2); 224 225 if (cpu_has_amu()) 226 __activate_fgt(hctxt, vcpu, HAFGRTR_EL2); 227 228 if (!cpus_have_final_cap(ARM64_HAS_FGT2)) 229 return; 230 231 __activate_fgt(hctxt, vcpu, HFGRTR2_EL2); 232 __activate_fgt(hctxt, vcpu, HFGWTR2_EL2); 233 __activate_fgt(hctxt, vcpu, HFGITR2_EL2); 234 __activate_fgt(hctxt, vcpu, HDFGRTR2_EL2); 235 __activate_fgt(hctxt, vcpu, HDFGWTR2_EL2); 236 } 237 238 static inline void __activate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu) 239 { 240 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 241 242 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 243 return; 244 245 __activate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2); 246 __activate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2); 247 __activate_fgt(hctxt, vcpu, ICH_HFGITR_EL2); 248 } 249 250 #define __deactivate_fgt(hctxt, vcpu, reg) \ 251 do { \ 252 write_sysreg_s(ctxt_sys_reg(hctxt, reg), \ 253 SYS_ ## reg); \ 254 } while(0) 255 256 static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) 257 { 258 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 259 260 if (!cpus_have_final_cap(ARM64_HAS_FGT)) 261 return; 262 263 __deactivate_fgt(hctxt, vcpu, HFGRTR_EL2); 264 __deactivate_fgt(hctxt, vcpu, HFGWTR_EL2); 265 __deactivate_fgt(hctxt, vcpu, HFGITR_EL2); 266 __deactivate_fgt(hctxt, vcpu, HDFGRTR_EL2); 267 __deactivate_fgt(hctxt, vcpu, HDFGWTR_EL2); 268 269 if (cpu_has_amu()) 270 __deactivate_fgt(hctxt, vcpu, HAFGRTR_EL2); 271 272 if (!cpus_have_final_cap(ARM64_HAS_FGT2)) 273 return; 274 275 __deactivate_fgt(hctxt, vcpu, HFGRTR2_EL2); 276 __deactivate_fgt(hctxt, vcpu, HFGWTR2_EL2); 277 __deactivate_fgt(hctxt, vcpu, HFGITR2_EL2); 278 __deactivate_fgt(hctxt, vcpu, HDFGRTR2_EL2); 279 __deactivate_fgt(hctxt, vcpu, HDFGWTR2_EL2); 280 } 281 282 static inline void __deactivate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu) 283 { 284 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 285 286 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 287 return; 288 289 __deactivate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2); 290 __deactivate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2); 291 __deactivate_fgt(hctxt, vcpu, ICH_HFGITR_EL2); 292 293 } 294 295 static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) 296 { 297 u64 clr = MPAM2_EL2_EnMPAMSM; 298 u64 set = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; 299 300 if (!system_supports_mpam()) 301 return; 302 303 /* trap guest access to MPAMIDR_EL1 */ 304 if (system_supports_mpam_hcr()) { 305 write_sysreg_s(MPAMHCR_EL2_TRAP_MPAMIDR_EL1, SYS_MPAMHCR_EL2); 306 } else { 307 /* From v1.1 TIDR can trap MPAMIDR, set it unconditionally */ 308 set |= MPAM2_EL2_TIDR; 309 } 310 311 sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); 312 } 313 314 static inline void __deactivate_traps_mpam(void) 315 { 316 u64 clr = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1 | MPAM2_EL2_TIDR; 317 u64 set = MPAM2_EL2_EnMPAMSM; 318 319 if (!system_supports_mpam()) 320 return; 321 322 sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); 323 324 if (system_supports_mpam_hcr()) 325 write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); 326 } 327 328 static inline void __activate_traps_common(struct kvm_vcpu *vcpu) 329 { 330 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 331 332 /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */ 333 write_sysreg(1 << 15, hstr_el2); 334 335 /* 336 * Make sure we trap PMU access from EL0 to EL2. Also sanitize 337 * PMSELR_EL0 to make sure it never contains the cycle 338 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at 339 * EL1 instead of being trapped to EL2. 340 */ 341 if (system_supports_pmuv3()) { 342 write_sysreg(0, pmselr_el0); 343 344 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = read_sysreg(pmuserenr_el0); 345 write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0); 346 vcpu_set_flag(vcpu, PMUSERENR_ON_CPU); 347 } 348 349 if (cpus_have_final_cap(ARM64_HAS_HCX)) { 350 u64 hcrx = vcpu->arch.hcrx_el2; 351 if (is_nested_ctxt(vcpu)) { 352 u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2); 353 hcrx |= val & __HCRX_EL2_MASK; 354 hcrx &= ~(~val & __HCRX_EL2_nMASK); 355 } 356 357 ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2); 358 write_sysreg_s(hcrx, SYS_HCRX_EL2); 359 } 360 361 __activate_traps_hfgxtr(vcpu); 362 __activate_traps_ich_hfgxtr(vcpu); 363 __activate_traps_mpam(vcpu); 364 } 365 366 static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) 367 { 368 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 369 370 write_sysreg(0, hstr_el2); 371 if (system_supports_pmuv3()) { 372 write_sysreg(ctxt_sys_reg(hctxt, PMUSERENR_EL0), pmuserenr_el0); 373 vcpu_clear_flag(vcpu, PMUSERENR_ON_CPU); 374 } 375 376 if (cpus_have_final_cap(ARM64_HAS_HCX)) 377 write_sysreg_s(ctxt_sys_reg(hctxt, HCRX_EL2), SYS_HCRX_EL2); 378 379 __deactivate_traps_hfgxtr(vcpu); 380 __deactivate_traps_ich_hfgxtr(vcpu); 381 __deactivate_traps_mpam(); 382 } 383 384 static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr) 385 { 386 if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) 387 hcr |= HCR_TVM; 388 389 write_sysreg_hcr(hcr); 390 391 if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) { 392 u64 vsesr; 393 394 /* 395 * When HCR_EL2.AMO is set, physical SErrors are taken to EL2 396 * and vSError injection is enabled for EL1. Conveniently, for 397 * NV this means that it is never the case where a 'physical' 398 * SError (injected by KVM or userspace) and vSError are 399 * deliverable to the same context. 400 * 401 * As such, we can trivially select between the host or guest's 402 * VSESR_EL2. Except for the case that FEAT_RAS hasn't been 403 * exposed to the guest, where ESR propagation in hardware 404 * occurs unconditionally. 405 * 406 * Paper over the architectural wart and use an IMPLEMENTATION 407 * DEFINED ESR value in case FEAT_RAS is hidden from the guest. 408 */ 409 if (!vserror_state_is_nested(vcpu)) 410 vsesr = vcpu->arch.vsesr_el2; 411 else if (kvm_has_ras(kern_hyp_va(vcpu->kvm))) 412 vsesr = __vcpu_sys_reg(vcpu, VSESR_EL2); 413 else 414 vsesr = ESR_ELx_ISV; 415 416 write_sysreg_s(vsesr, SYS_VSESR_EL2); 417 } 418 } 419 420 static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) 421 { 422 u64 *hcr; 423 424 if (vserror_state_is_nested(vcpu)) 425 hcr = __ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2); 426 else 427 hcr = &vcpu->arch.hcr_el2; 428 429 /* 430 * If we pended a virtual abort, preserve it until it gets 431 * cleared. See D1.14.3 (Virtual Interrupts) for details, but 432 * the crucial bit is "On taking a vSError interrupt, 433 * HCR_EL2.VSE is cleared to 0." 434 * 435 * Additionally, when in a nested context we need to propagate the 436 * updated state to the guest hypervisor's HCR_EL2. 437 */ 438 if (*hcr & HCR_VSE) { 439 *hcr &= ~HCR_VSE; 440 *hcr |= read_sysreg(hcr_el2) & HCR_VSE; 441 } 442 } 443 444 static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) 445 { 446 return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault); 447 } 448 449 static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) 450 { 451 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); 452 arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); 453 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); 454 455 /* 456 * Finish potential single step before executing the prologue 457 * instruction. 458 */ 459 *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; 460 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 461 462 return true; 463 } 464 465 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) 466 { 467 u64 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 468 469 /* 470 * The vCPU's saved SVE state layout always matches the max VL of the 471 * vCPU. Start off with the max VL so we can load the SVE state. 472 */ 473 sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); 474 sve_load_state(kern_hyp_va(vcpu->arch.sve_state), true); 475 fpsimd_load_common(&vcpu->arch.ctxt.fp_regs); 476 477 /* 478 * The effective VL for a VM could differ from the max VL when running a 479 * nested guest, as the guest hypervisor could select a smaller VL. Slap 480 * that into hardware before wrapping up. 481 */ 482 if (is_nested_ctxt(vcpu)) { 483 zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2)); 484 sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2); 485 } 486 487 write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR); 488 } 489 490 static inline void __hyp_sve_save_host(void) 491 { 492 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 493 struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); 494 495 ctxt_sys_reg(hctxt, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); 496 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 497 sve_save_state(sve_regs, true); 498 fpsimd_save_common(&hctxt->fp_regs); 499 } 500 501 static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) 502 { 503 u64 zcr_el1, zcr_el2; 504 505 if (!guest_owns_fp_regs()) 506 return; 507 508 if (vcpu_has_sve(vcpu)) { 509 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 510 511 /* A guest hypervisor may restrict the effective max VL. */ 512 if (is_nested_ctxt(vcpu)) 513 zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2)); 514 515 write_sysreg_el2(zcr_el2, SYS_ZCR); 516 517 zcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)); 518 write_sysreg_el1(zcr_el1, SYS_ZCR); 519 } 520 } 521 522 static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) 523 { 524 u64 zcr_el1, zcr_el2; 525 526 if (!guest_owns_fp_regs()) 527 return; 528 529 /* 530 * When the guest owns the FP regs, we know that guest+hyp traps for 531 * any FPSIMD/SVE/SME features exposed to the guest have been disabled 532 * by either __activate_cptr_traps() or kvm_hyp_handle_fpsimd() 533 * prior to __guest_entry(). As __guest_entry() guarantees a context 534 * synchronization event, we don't need an ISB here to avoid taking 535 * traps for anything that was exposed to the guest. 536 */ 537 if (vcpu_has_sve(vcpu)) { 538 zcr_el1 = read_sysreg_el1(SYS_ZCR); 539 __vcpu_assign_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu), zcr_el1); 540 541 /* 542 * The guest's state is always saved using the guest's max VL. 543 * Ensure that the host has the guest's max VL active such that 544 * the host can save the guest's state lazily, but don't 545 * artificially restrict the host to the guest's max VL. 546 */ 547 if (has_vhe()) { 548 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; 549 write_sysreg_el2(zcr_el2, SYS_ZCR); 550 } else { 551 zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; 552 write_sysreg_el2(zcr_el2, SYS_ZCR); 553 554 zcr_el1 = vcpu_sve_max_vq(vcpu) - 1; 555 write_sysreg_el1(zcr_el1, SYS_ZCR); 556 } 557 } 558 } 559 560 static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu) 561 { 562 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 563 564 /* 565 * Non-protected kvm relies on the host restoring its sve state. 566 * Protected kvm restores the host's sve state as not to reveal that 567 * fpsimd was used by a guest nor leak upper sve bits. 568 */ 569 if (system_supports_sve()) { 570 __hyp_sve_save_host(); 571 } else { 572 fpsimd_save_state(&hctxt->fp_regs); 573 } 574 575 if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) 576 ctxt_sys_reg(hctxt, FPMR) = read_sysreg_s(SYS_FPMR); 577 } 578 579 580 /* 581 * We trap the first access to the FP/SIMD to save the host context and 582 * restore the guest context lazily. 583 * If FP/SIMD is not implemented, handle the trap and inject an undefined 584 * instruction exception to the guest. Similarly for trapped SVE accesses. 585 */ 586 static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) 587 { 588 bool sve_guest; 589 u8 esr_ec; 590 591 if (!system_supports_fpsimd()) 592 return false; 593 594 sve_guest = vcpu_has_sve(vcpu); 595 esr_ec = kvm_vcpu_trap_get_class(vcpu); 596 597 /* Only handle traps the vCPU can support here: */ 598 switch (esr_ec) { 599 case ESR_ELx_EC_FP_ASIMD: 600 /* Forward traps to the guest hypervisor as required */ 601 if (guest_hyp_fpsimd_traps_enabled(vcpu)) 602 return false; 603 break; 604 case ESR_ELx_EC_SYS64: 605 if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) 606 return false; 607 fallthrough; 608 case ESR_ELx_EC_SVE: 609 if (!sve_guest) 610 return false; 611 if (guest_hyp_sve_traps_enabled(vcpu)) 612 return false; 613 break; 614 default: 615 return false; 616 } 617 618 /* Valid trap. Switch the context: */ 619 620 /* First disable enough traps to allow us to update the registers */ 621 __deactivate_cptr_traps(vcpu); 622 isb(); 623 624 /* Write out the host state if it's in the registers */ 625 if (is_protected_kvm_enabled() && host_owns_fp_regs()) 626 kvm_hyp_save_fpsimd_host(vcpu); 627 628 /* Restore the guest state */ 629 if (sve_guest) 630 __hyp_sve_restore_guest(vcpu); 631 else 632 fpsimd_load_state(&vcpu->arch.ctxt.fp_regs); 633 634 if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) 635 write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); 636 637 /* Skip restoring fpexc32 for AArch64 guests */ 638 if (!(read_sysreg(hcr_el2) & HCR_RW)) 639 write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); 640 641 *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; 642 643 /* 644 * Re-enable traps necessary for the current state of the guest, e.g. 645 * those enabled by a guest hypervisor. The ERET to the guest will 646 * provide the necessary context synchronization. 647 */ 648 __activate_cptr_traps(vcpu); 649 650 return true; 651 } 652 653 static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) 654 { 655 u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 656 int rt = kvm_vcpu_sys_get_rt(vcpu); 657 u64 val = vcpu_get_reg(vcpu, rt); 658 659 /* 660 * The normal sysreg handling code expects to see the traps, 661 * let's not do anything here. 662 */ 663 if (vcpu->arch.hcr_el2 & HCR_TVM) 664 return false; 665 666 switch (sysreg) { 667 case SYS_SCTLR_EL1: 668 write_sysreg_el1(val, SYS_SCTLR); 669 break; 670 case SYS_TTBR0_EL1: 671 write_sysreg_el1(val, SYS_TTBR0); 672 break; 673 case SYS_TTBR1_EL1: 674 write_sysreg_el1(val, SYS_TTBR1); 675 break; 676 case SYS_TCR_EL1: 677 write_sysreg_el1(val, SYS_TCR); 678 break; 679 case SYS_ESR_EL1: 680 write_sysreg_el1(val, SYS_ESR); 681 break; 682 case SYS_FAR_EL1: 683 write_sysreg_el1(val, SYS_FAR); 684 break; 685 case SYS_AFSR0_EL1: 686 write_sysreg_el1(val, SYS_AFSR0); 687 break; 688 case SYS_AFSR1_EL1: 689 write_sysreg_el1(val, SYS_AFSR1); 690 break; 691 case SYS_MAIR_EL1: 692 write_sysreg_el1(val, SYS_MAIR); 693 break; 694 case SYS_AMAIR_EL1: 695 write_sysreg_el1(val, SYS_AMAIR); 696 break; 697 case SYS_CONTEXTIDR_EL1: 698 write_sysreg_el1(val, SYS_CONTEXTIDR); 699 break; 700 default: 701 return false; 702 } 703 704 __kvm_skip_instr(vcpu); 705 return true; 706 } 707 708 /* Open-coded version of timer_get_offset() to allow for kern_hyp_va() */ 709 static inline u64 hyp_timer_get_offset(struct arch_timer_context *ctxt) 710 { 711 u64 offset = 0; 712 713 if (ctxt->offset.vm_offset) 714 offset += *kern_hyp_va(ctxt->offset.vm_offset); 715 if (ctxt->offset.vcpu_offset) 716 offset += *kern_hyp_va(ctxt->offset.vcpu_offset); 717 718 return offset; 719 } 720 721 static inline u64 compute_counter_value(struct arch_timer_context *ctxt) 722 { 723 return arch_timer_read_cntpct_el0() - hyp_timer_get_offset(ctxt); 724 } 725 726 static bool kvm_handle_cntxct(struct kvm_vcpu *vcpu) 727 { 728 struct arch_timer_context *ctxt; 729 u32 sysreg; 730 u64 val; 731 732 /* 733 * We only get here for 64bit guests, 32bit guests will hit 734 * the long and winding road all the way to the standard 735 * handling. Yes, it sucks to be irrelevant. 736 * 737 * Also, we only deal with non-hypervisor context here (either 738 * an EL1 guest, or a non-HYP context of an EL2 guest). 739 */ 740 if (is_hyp_ctxt(vcpu)) 741 return false; 742 743 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 744 745 switch (sysreg) { 746 case SYS_CNTPCT_EL0: 747 case SYS_CNTPCTSS_EL0: 748 if (vcpu_has_nv(vcpu)) { 749 /* Check for guest hypervisor trapping */ 750 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2); 751 if (!vcpu_el2_e2h_is_set(vcpu)) 752 val = (val & CNTHCTL_EL1PCTEN) << 10; 753 754 if (!(val & (CNTHCTL_EL1PCTEN << 10))) 755 return false; 756 } 757 758 ctxt = vcpu_ptimer(vcpu); 759 break; 760 case SYS_CNTVCT_EL0: 761 case SYS_CNTVCTSS_EL0: 762 if (vcpu_has_nv(vcpu)) { 763 /* Check for guest hypervisor trapping */ 764 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2); 765 766 if (val & CNTHCTL_EL1TVCT) 767 return false; 768 } 769 770 ctxt = vcpu_vtimer(vcpu); 771 break; 772 default: 773 return false; 774 } 775 776 val = compute_counter_value(ctxt); 777 778 vcpu_set_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu), val); 779 __kvm_skip_instr(vcpu); 780 return true; 781 } 782 783 static bool handle_ampere1_tcr(struct kvm_vcpu *vcpu) 784 { 785 u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); 786 int rt = kvm_vcpu_sys_get_rt(vcpu); 787 u64 val = vcpu_get_reg(vcpu, rt); 788 789 if (sysreg != SYS_TCR_EL1) 790 return false; 791 792 /* 793 * Affected parts do not advertise support for hardware Access Flag / 794 * Dirty state management in ID_AA64MMFR1_EL1.HAFDBS, but the underlying 795 * control bits are still functional. The architecture requires these be 796 * RES0 on systems that do not implement FEAT_HAFDBS. 797 * 798 * Uphold the requirements of the architecture by masking guest writes 799 * to TCR_EL1.{HA,HD} here. 800 */ 801 val &= ~(TCR_HD | TCR_HA); 802 write_sysreg_el1(val, SYS_TCR); 803 __kvm_skip_instr(vcpu); 804 return true; 805 } 806 807 static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) 808 { 809 if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && 810 handle_tx2_tvm(vcpu)) 811 return true; 812 813 if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38) && 814 handle_ampere1_tcr(vcpu)) 815 return true; 816 817 if (static_branch_unlikely(&vgic_v3_cpuif_trap) && 818 __vgic_v3_perform_cpuif_access(vcpu) == 1) 819 return true; 820 821 if (kvm_handle_cntxct(vcpu)) 822 return true; 823 824 return false; 825 } 826 827 static inline bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) 828 { 829 if (static_branch_unlikely(&vgic_v3_cpuif_trap) && 830 __vgic_v3_perform_cpuif_access(vcpu) == 1) 831 return true; 832 833 return false; 834 } 835 836 static inline bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu, 837 u64 *exit_code) 838 { 839 if (!__populate_fault_info(vcpu)) 840 return true; 841 842 return false; 843 } 844 #define kvm_hyp_handle_iabt_low kvm_hyp_handle_memory_fault 845 #define kvm_hyp_handle_watchpt_low kvm_hyp_handle_memory_fault 846 847 static inline bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) 848 { 849 if (kvm_hyp_handle_memory_fault(vcpu, exit_code)) 850 return true; 851 852 if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { 853 bool valid; 854 855 valid = kvm_vcpu_trap_is_translation_fault(vcpu) && 856 kvm_vcpu_dabt_isvalid(vcpu) && 857 !kvm_vcpu_abt_issea(vcpu) && 858 !kvm_vcpu_abt_iss1tw(vcpu); 859 860 if (valid) { 861 int ret = __vgic_v2_perform_cpuif_access(vcpu); 862 863 if (ret == 1) 864 return true; 865 866 /* Promote an illegal access to an SError.*/ 867 if (ret == -1) 868 *exit_code = ARM_EXCEPTION_EL1_SERROR; 869 } 870 } 871 872 return false; 873 } 874 875 typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *); 876 877 /* 878 * Allow the hypervisor to handle the exit with an exit handler if it has one. 879 * 880 * Returns true if the hypervisor handled the exit, and control should go back 881 * to the guest, or false if it hasn't. 882 */ 883 static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code, 884 const exit_handler_fn *handlers) 885 { 886 exit_handler_fn fn = handlers[kvm_vcpu_trap_get_class(vcpu)]; 887 if (fn) 888 return fn(vcpu, exit_code); 889 890 return false; 891 } 892 893 static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu) 894 { 895 /* 896 * Check for the conditions of Cortex-A510's #2077057. When these occur 897 * SPSR_EL2 can't be trusted, but isn't needed either as it is 898 * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. 899 * Are we single-stepping the guest, and took a PAC exception from the 900 * active-not-pending state? 901 */ 902 if (cpus_have_final_cap(ARM64_WORKAROUND_2077057) && 903 vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 904 *vcpu_cpsr(vcpu) & DBG_SPSR_SS && 905 ESR_ELx_EC(read_sysreg_el2(SYS_ESR)) == ESR_ELx_EC_PAC) 906 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); 907 908 vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR); 909 } 910 911 /* 912 * Return true when we were able to fixup the guest exit and should return to 913 * the guest, false when we should restore the host state and return to the 914 * main run loop. 915 */ 916 static inline bool __fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code, 917 const exit_handler_fn *handlers) 918 { 919 if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) 920 vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); 921 922 if (ARM_SERROR_PENDING(*exit_code) && 923 ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) { 924 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu); 925 926 /* 927 * HVC already have an adjusted PC, which we need to 928 * correct in order to return to after having injected 929 * the SError. 930 * 931 * SMC, on the other hand, is *trapped*, meaning its 932 * preferred return address is the SMC itself. 933 */ 934 if (esr_ec == ESR_ELx_EC_HVC32 || esr_ec == ESR_ELx_EC_HVC64) 935 write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR); 936 } 937 938 /* 939 * We're using the raw exception code in order to only process 940 * the trap if no SError is pending. We will come back to the 941 * same PC once the SError has been injected, and replay the 942 * trapping instruction. 943 */ 944 if (*exit_code != ARM_EXCEPTION_TRAP) 945 goto exit; 946 947 /* Check if there's an exit handler and allow it to handle the exit. */ 948 if (kvm_hyp_handle_exit(vcpu, exit_code, handlers)) 949 goto guest; 950 exit: 951 /* Return to the host kernel and handle the exit */ 952 return false; 953 954 guest: 955 /* Re-enter the guest */ 956 asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); 957 return true; 958 } 959 960 static inline void __kvm_unexpected_el2_exception(void) 961 { 962 extern char __guest_exit_restore_elr_and_panic[]; 963 unsigned long addr, fixup; 964 struct kvm_exception_table_entry *entry, *end; 965 unsigned long elr_el2 = read_sysreg(elr_el2); 966 967 entry = &__start___kvm_ex_table; 968 end = &__stop___kvm_ex_table; 969 970 while (entry < end) { 971 addr = (unsigned long)&entry->insn + entry->insn; 972 fixup = (unsigned long)&entry->fixup + entry->fixup; 973 974 if (addr != elr_el2) { 975 entry++; 976 continue; 977 } 978 979 write_sysreg(fixup, elr_el2); 980 return; 981 } 982 983 /* Trigger a panic after restoring the hyp context. */ 984 this_cpu_ptr(&kvm_hyp_ctxt)->sys_regs[ELR_EL2] = elr_el2; 985 write_sysreg(__guest_exit_restore_elr_and_panic, elr_el2); 986 } 987 988 #endif /* __ARM64_KVM_HYP_SWITCH_H__ */ 989