Lines Matching full:vcpu
52 static u64 __compute_hcr(struct kvm_vcpu *vcpu) in __compute_hcr() argument
54 u64 guest_hcr, hcr = vcpu->arch.hcr_el2; in __compute_hcr()
56 if (!vcpu_has_nv(vcpu)) in __compute_hcr()
60 * We rely on the invariant that a vcpu entered from HYP in __compute_hcr()
65 if (is_hyp_ctxt(vcpu)) { in __compute_hcr()
70 if (!vcpu_el2_e2h_is_set(vcpu)) in __compute_hcr()
81 guest_hcr = kvm_vcpu_apply_reg_masks(vcpu, HCR_EL2, 0); in __compute_hcr()
83 write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2); in __compute_hcr()
87 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2); in __compute_hcr()
92 va |= __vcpu_sys_reg(vcpu, VNCR_EL2) & GENMASK(PAGE_SHIFT - 1, 0); in __compute_hcr()
113 static void __activate_traps(struct kvm_vcpu *vcpu) in __activate_traps() argument
117 ___activate_traps(vcpu, __compute_hcr(vcpu)); in __activate_traps()
122 get_timer_map(vcpu, &map); in __activate_traps()
128 if (map.direct_ptimer == vcpu_ptimer(vcpu)) in __activate_traps()
129 val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); in __activate_traps()
130 if (map.direct_ptimer == vcpu_hptimer(vcpu)) in __activate_traps()
131 val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); in __activate_traps()
139 __activate_cptr_traps(vcpu); in __activate_traps()
145 static void __deactivate_traps(struct kvm_vcpu *vcpu) in __deactivate_traps() argument
149 ___deactivate_traps(vcpu); in __deactivate_traps()
157 get_timer_map(vcpu, &map); in __deactivate_traps()
164 if (map.direct_ptimer == vcpu_ptimer(vcpu)) in __deactivate_traps()
165 __vcpu_assign_sys_reg(vcpu, CNTP_CVAL_EL0, val); in __deactivate_traps()
166 if (map.direct_ptimer == vcpu_hptimer(vcpu)) in __deactivate_traps()
167 __vcpu_assign_sys_reg(vcpu, CNTHP_CVAL_EL2, val); in __deactivate_traps()
184 __deactivate_cptr_traps(vcpu); in __deactivate_traps()
198 static void __vcpu_load_activate_traps(struct kvm_vcpu *vcpu) in __vcpu_load_activate_traps() argument
203 __activate_traps_common(vcpu); in __vcpu_load_activate_traps()
207 static void __vcpu_put_deactivate_traps(struct kvm_vcpu *vcpu) in __vcpu_put_deactivate_traps() argument
212 __deactivate_traps_common(vcpu); in __vcpu_put_deactivate_traps()
216 void kvm_vcpu_load_vhe(struct kvm_vcpu *vcpu) in kvm_vcpu_load_vhe() argument
218 host_data_ptr(host_ctxt)->__hyp_running_vcpu = vcpu; in kvm_vcpu_load_vhe()
220 __vcpu_load_switch_sysregs(vcpu); in kvm_vcpu_load_vhe()
221 __vcpu_load_activate_traps(vcpu); in kvm_vcpu_load_vhe()
222 __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch); in kvm_vcpu_load_vhe()
225 void kvm_vcpu_put_vhe(struct kvm_vcpu *vcpu) in kvm_vcpu_put_vhe() argument
227 __vcpu_put_deactivate_traps(vcpu); in kvm_vcpu_put_vhe()
228 __vcpu_put_switch_sysregs(vcpu); in kvm_vcpu_put_vhe()
233 static u64 compute_emulated_cntx_ctl_el0(struct kvm_vcpu *vcpu, in compute_emulated_cntx_ctl_el0() argument
242 cval = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); in compute_emulated_cntx_ctl_el0()
243 ctl = __vcpu_sys_reg(vcpu, CNTP_CTL_EL0); in compute_emulated_cntx_ctl_el0()
244 cnt = compute_counter_value(vcpu_ptimer(vcpu)); in compute_emulated_cntx_ctl_el0()
247 cval = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); in compute_emulated_cntx_ctl_el0()
248 ctl = __vcpu_sys_reg(vcpu, CNTV_CTL_EL0); in compute_emulated_cntx_ctl_el0()
249 cnt = compute_counter_value(vcpu_vtimer(vcpu)); in compute_emulated_cntx_ctl_el0()
261 static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_timer() argument
271 if (!is_hyp_ctxt(vcpu)) in kvm_hyp_handle_timer()
274 esr = kvm_vcpu_get_esr(vcpu); in kvm_hyp_handle_timer()
280 val = compute_emulated_cntx_ctl_el0(vcpu, CNTP_CTL_EL0); in kvm_hyp_handle_timer()
283 if (vcpu_el2_e2h_is_set(vcpu)) in kvm_hyp_handle_timer()
286 val = compute_emulated_cntx_ctl_el0(vcpu, CNTP_CTL_EL0); in kvm_hyp_handle_timer()
289 val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); in kvm_hyp_handle_timer()
292 if (vcpu_el2_e2h_is_set(vcpu)) { in kvm_hyp_handle_timer()
296 val -= timer_get_offset(vcpu_hptimer(vcpu)); in kvm_hyp_handle_timer()
298 val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); in kvm_hyp_handle_timer()
303 val = compute_counter_value(vcpu_hptimer(vcpu)); in kvm_hyp_handle_timer()
306 val = compute_emulated_cntx_ctl_el0(vcpu, CNTV_CTL_EL0); in kvm_hyp_handle_timer()
309 if (vcpu_el2_e2h_is_set(vcpu)) in kvm_hyp_handle_timer()
312 val = compute_emulated_cntx_ctl_el0(vcpu, CNTV_CTL_EL0); in kvm_hyp_handle_timer()
315 val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); in kvm_hyp_handle_timer()
318 if (vcpu_el2_e2h_is_set(vcpu)) in kvm_hyp_handle_timer()
321 val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); in kvm_hyp_handle_timer()
325 val = compute_counter_value(vcpu_hvtimer(vcpu)); in kvm_hyp_handle_timer()
331 vcpu_set_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu), val); in kvm_hyp_handle_timer()
332 __kvm_skip_instr(vcpu); in kvm_hyp_handle_timer()
337 static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_eret() argument
339 u64 esr = kvm_vcpu_get_esr(vcpu); in kvm_hyp_handle_eret()
352 if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) || in kvm_hyp_handle_eret()
353 (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_ERET)) in kvm_hyp_handle_eret()
361 if (!(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))) in kvm_hyp_handle_eret()
376 if (!(vcpu_has_ptrauth(vcpu) && kvm_auth_eretax(vcpu, &elr))) in kvm_hyp_handle_eret()
390 static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_tlbi_el2() argument
412 if (!(is_hyp_ctxt(vcpu))) in kvm_hyp_handle_tlbi_el2()
415 instr = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); in kvm_hyp_handle_tlbi_el2()
416 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); in kvm_hyp_handle_tlbi_el2()
418 if ((kvm_supported_tlbi_s1e1_op(vcpu, instr) && in kvm_hyp_handle_tlbi_el2()
419 vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) || in kvm_hyp_handle_tlbi_el2()
420 kvm_supported_tlbi_s1e2_op (vcpu, instr)) in kvm_hyp_handle_tlbi_el2()
430 if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && in kvm_hyp_handle_tlbi_el2()
431 atomic_read(&vcpu->kvm->arch.vncr_map_count)) in kvm_hyp_handle_tlbi_el2()
434 __kvm_skip_instr(vcpu); in kvm_hyp_handle_tlbi_el2()
439 static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_cpacr_el1() argument
441 u64 esr = kvm_vcpu_get_esr(vcpu); in kvm_hyp_handle_cpacr_el1()
444 if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1) in kvm_hyp_handle_cpacr_el1()
447 rt = kvm_vcpu_sys_get_rt(vcpu); in kvm_hyp_handle_cpacr_el1()
450 vcpu_set_reg(vcpu, rt, __vcpu_sys_reg(vcpu, CPTR_EL2)); in kvm_hyp_handle_cpacr_el1()
452 vcpu_write_sys_reg(vcpu, vcpu_get_reg(vcpu, rt), CPTR_EL2); in kvm_hyp_handle_cpacr_el1()
453 __activate_cptr_traps(vcpu); in kvm_hyp_handle_cpacr_el1()
456 __kvm_skip_instr(vcpu); in kvm_hyp_handle_cpacr_el1()
461 static bool kvm_hyp_handle_zcr_el2(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_zcr_el2() argument
463 u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu)); in kvm_hyp_handle_zcr_el2()
465 if (!vcpu_has_nv(vcpu)) in kvm_hyp_handle_zcr_el2()
481 kvm_hyp_handle_fpsimd(vcpu, exit_code); in kvm_hyp_handle_zcr_el2()
485 static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_sysreg_vhe() argument
487 if (kvm_hyp_handle_tlbi_el2(vcpu, exit_code)) in kvm_hyp_handle_sysreg_vhe()
490 if (kvm_hyp_handle_timer(vcpu, exit_code)) in kvm_hyp_handle_sysreg_vhe()
493 if (kvm_hyp_handle_cpacr_el1(vcpu, exit_code)) in kvm_hyp_handle_sysreg_vhe()
496 if (kvm_hyp_handle_zcr_el2(vcpu, exit_code)) in kvm_hyp_handle_sysreg_vhe()
499 return kvm_hyp_handle_sysreg(vcpu, exit_code); in kvm_hyp_handle_sysreg_vhe()
502 static bool kvm_hyp_handle_impdef(struct kvm_vcpu *vcpu, u64 *exit_code) in kvm_hyp_handle_impdef() argument
515 vcpu->arch.fault.esr_el2 = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SYS64) | in kvm_hyp_handle_impdef()
537 static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) in fixup_guest_exit() argument
539 synchronize_vcpu_pstate(vcpu, exit_code); in fixup_guest_exit()
547 if (vcpu_has_nv(vcpu) && in fixup_guest_exit()
549 u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); in fixup_guest_exit()
560 *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT); in fixup_guest_exit()
561 *vcpu_cpsr(vcpu) |= mode; in fixup_guest_exit()
565 BUG_ON(vcpu_has_nv(vcpu) && in fixup_guest_exit()
566 !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); in fixup_guest_exit()
568 return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); in fixup_guest_exit()
572 static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) in __kvm_vcpu_run_vhe() argument
579 guest_ctxt = &vcpu->arch.ctxt; in __kvm_vcpu_run_vhe()
581 fpsimd_lazy_switch_to_guest(vcpu); in __kvm_vcpu_run_vhe()
591 __activate_traps(vcpu); in __kvm_vcpu_run_vhe()
593 __kvm_adjust_pc(vcpu); in __kvm_vcpu_run_vhe()
596 __debug_switch_to_guest(vcpu); in __kvm_vcpu_run_vhe()
600 exit_code = __guest_enter(vcpu); in __kvm_vcpu_run_vhe()
603 } while (fixup_guest_exit(vcpu, &exit_code)); in __kvm_vcpu_run_vhe()
607 __deactivate_traps(vcpu); in __kvm_vcpu_run_vhe()
611 __debug_switch_to_host(vcpu); in __kvm_vcpu_run_vhe()
621 fpsimd_lazy_switch_to_host(vcpu); in __kvm_vcpu_run_vhe()
624 __fpsimd_save_fpexc32(vcpu); in __kvm_vcpu_run_vhe()
630 int __kvm_vcpu_run(struct kvm_vcpu *vcpu) in __kvm_vcpu_run() argument
647 ret = __kvm_vcpu_run_vhe(vcpu); in __kvm_vcpu_run()
661 struct kvm_vcpu *vcpu; in __hyp_call_panic() local
664 vcpu = host_ctxt->__hyp_running_vcpu; in __hyp_call_panic()
666 __deactivate_traps(vcpu); in __hyp_call_panic()
672 read_sysreg(hpfar_el2), par, vcpu); in __hyp_call_panic()