1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <hyp/switch.h>
8 #include <hyp/sysreg-sr.h>
9
10 #include <linux/arm-smccc.h>
11 #include <linux/kvm_host.h>
12 #include <linux/types.h>
13 #include <linux/jump_label.h>
14 #include <uapi/linux/psci.h>
15
16 #include <kvm/arm_psci.h>
17
18 #include <asm/barrier.h>
19 #include <asm/cpufeature.h>
20 #include <asm/kprobes.h>
21 #include <asm/kvm_asm.h>
22 #include <asm/kvm_emulate.h>
23 #include <asm/kvm_hyp.h>
24 #include <asm/kvm_mmu.h>
25 #include <asm/fpsimd.h>
26 #include <asm/debug-monitors.h>
27 #include <asm/processor.h>
28
29 #include <nvhe/mem_protect.h>
30
31 /* Non-VHE specific context */
32 DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
33 DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
34 DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
35
36 extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc);
37
__activate_cptr_traps(struct kvm_vcpu * vcpu)38 static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
39 {
40 u64 val = CPTR_EL2_TAM; /* Same bit irrespective of E2H */
41
42 if (!guest_owns_fp_regs())
43 __activate_traps_fpsimd32(vcpu);
44
45 if (has_hvhe()) {
46 val |= CPACR_EL1_TTA;
47
48 if (guest_owns_fp_regs()) {
49 val |= CPACR_EL1_FPEN;
50 if (vcpu_has_sve(vcpu))
51 val |= CPACR_EL1_ZEN;
52 }
53
54 write_sysreg(val, cpacr_el1);
55 } else {
56 val |= CPTR_EL2_TTA | CPTR_NVHE_EL2_RES1;
57
58 /*
59 * Always trap SME since it's not supported in KVM.
60 * TSM is RES1 if SME isn't implemented.
61 */
62 val |= CPTR_EL2_TSM;
63
64 if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs())
65 val |= CPTR_EL2_TZ;
66
67 if (!guest_owns_fp_regs())
68 val |= CPTR_EL2_TFP;
69
70 write_sysreg(val, cptr_el2);
71 }
72 }
73
__deactivate_cptr_traps(struct kvm_vcpu * vcpu)74 static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)
75 {
76 if (has_hvhe()) {
77 u64 val = CPACR_EL1_FPEN;
78
79 if (cpus_have_final_cap(ARM64_SVE))
80 val |= CPACR_EL1_ZEN;
81 if (cpus_have_final_cap(ARM64_SME))
82 val |= CPACR_EL1_SMEN;
83
84 write_sysreg(val, cpacr_el1);
85 } else {
86 u64 val = CPTR_NVHE_EL2_RES1;
87
88 if (!cpus_have_final_cap(ARM64_SVE))
89 val |= CPTR_EL2_TZ;
90 if (!cpus_have_final_cap(ARM64_SME))
91 val |= CPTR_EL2_TSM;
92
93 write_sysreg(val, cptr_el2);
94 }
95 }
96
__activate_traps(struct kvm_vcpu * vcpu)97 static void __activate_traps(struct kvm_vcpu *vcpu)
98 {
99 ___activate_traps(vcpu, vcpu->arch.hcr_el2);
100 __activate_traps_common(vcpu);
101 __activate_cptr_traps(vcpu);
102
103 write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2);
104
105 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
106 struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
107
108 isb();
109 /*
110 * At this stage, and thanks to the above isb(), S2 is
111 * configured and enabled. We can now restore the guest's S1
112 * configuration: SCTLR, and only then TCR.
113 */
114 write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
115 isb();
116 write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
117 }
118 }
119
__deactivate_traps(struct kvm_vcpu * vcpu)120 static void __deactivate_traps(struct kvm_vcpu *vcpu)
121 {
122 extern char __kvm_hyp_host_vector[];
123
124 ___deactivate_traps(vcpu);
125
126 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
127 u64 val;
128
129 /*
130 * Set the TCR and SCTLR registers in the exact opposite
131 * sequence as __activate_traps (first prevent walks,
132 * then force the MMU on). A generous sprinkling of isb()
133 * ensure that things happen in this exact order.
134 */
135 val = read_sysreg_el1(SYS_TCR);
136 write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR);
137 isb();
138 val = read_sysreg_el1(SYS_SCTLR);
139 write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR);
140 isb();
141 }
142
143 __deactivate_traps_common(vcpu);
144
145 write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2);
146
147 __deactivate_cptr_traps(vcpu);
148 write_sysreg(__kvm_hyp_host_vector, vbar_el2);
149 }
150
151 /* Save VGICv3 state on non-VHE systems */
__hyp_vgic_save_state(struct kvm_vcpu * vcpu)152 static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
153 {
154 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
155 __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
156 __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
157 }
158 }
159
160 /* Restore VGICv3 state on non-VHE systems */
__hyp_vgic_restore_state(struct kvm_vcpu * vcpu)161 static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
162 {
163 if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
164 __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
165 __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
166 }
167 }
168
169 /*
170 * Disable host events, enable guest events
171 */
172 #ifdef CONFIG_HW_PERF_EVENTS
__pmu_switch_to_guest(struct kvm_vcpu * vcpu)173 static bool __pmu_switch_to_guest(struct kvm_vcpu *vcpu)
174 {
175 struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events;
176
177 if (pmu->events_host)
178 write_sysreg(pmu->events_host, pmcntenclr_el0);
179
180 if (pmu->events_guest)
181 write_sysreg(pmu->events_guest, pmcntenset_el0);
182
183 return (pmu->events_host || pmu->events_guest);
184 }
185
186 /*
187 * Disable guest events, enable host events
188 */
__pmu_switch_to_host(struct kvm_vcpu * vcpu)189 static void __pmu_switch_to_host(struct kvm_vcpu *vcpu)
190 {
191 struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events;
192
193 if (pmu->events_guest)
194 write_sysreg(pmu->events_guest, pmcntenclr_el0);
195
196 if (pmu->events_host)
197 write_sysreg(pmu->events_host, pmcntenset_el0);
198 }
199 #else
200 #define __pmu_switch_to_guest(v) ({ false; })
201 #define __pmu_switch_to_host(v) do {} while (0)
202 #endif
203
204 /*
205 * Handler for protected VM MSR, MRS or System instruction execution in AArch64.
206 *
207 * Returns true if the hypervisor has handled the exit, and control should go
208 * back to the guest, or false if it hasn't.
209 */
kvm_handle_pvm_sys64(struct kvm_vcpu * vcpu,u64 * exit_code)210 static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code)
211 {
212 /*
213 * Make sure we handle the exit for workarounds before the pKVM
214 * handling, as the latter could decide to UNDEF.
215 */
216 return (kvm_hyp_handle_sysreg(vcpu, exit_code) ||
217 kvm_handle_pvm_sysreg(vcpu, exit_code));
218 }
219
220 static const exit_handler_fn hyp_exit_handlers[] = {
221 [0 ... ESR_ELx_EC_MAX] = NULL,
222 [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
223 [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg,
224 [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
225 [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
226 [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
227 [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
228 [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low,
229 [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops,
230 };
231
232 static const exit_handler_fn pvm_exit_handlers[] = {
233 [0 ... ESR_ELx_EC_MAX] = NULL,
234 [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64,
235 [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted,
236 [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
237 [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
238 [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
239 [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low,
240 [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops,
241 };
242
kvm_get_exit_handler_array(struct kvm_vcpu * vcpu)243 static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
244 {
245 if (unlikely(vcpu_is_protected(vcpu)))
246 return pvm_exit_handlers;
247
248 return hyp_exit_handlers;
249 }
250
fixup_guest_exit(struct kvm_vcpu * vcpu,u64 * exit_code)251 static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
252 {
253 const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu);
254
255 synchronize_vcpu_pstate(vcpu, exit_code);
256
257 /*
258 * Some guests (e.g., protected VMs) are not be allowed to run in
259 * AArch32. The ARMv8 architecture does not give the hypervisor a
260 * mechanism to prevent a guest from dropping to AArch32 EL0 if
261 * implemented by the CPU. If the hypervisor spots a guest in such a
262 * state ensure it is handled, and don't trust the host to spot or fix
263 * it. The check below is based on the one in
264 * kvm_arch_vcpu_ioctl_run().
265 */
266 if (unlikely(vcpu_is_protected(vcpu) && vcpu_mode_is_32bit(vcpu))) {
267 /*
268 * As we have caught the guest red-handed, decide that it isn't
269 * fit for purpose anymore by making the vcpu invalid. The VMM
270 * can try and fix it by re-initializing the vcpu with
271 * KVM_ARM_VCPU_INIT, however, this is likely not possible for
272 * protected VMs.
273 */
274 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
275 *exit_code &= BIT(ARM_EXIT_WITH_SERROR_BIT);
276 *exit_code |= ARM_EXCEPTION_IL;
277 }
278
279 return __fixup_guest_exit(vcpu, exit_code, handlers);
280 }
281
282 /* Switch to the guest for legacy non-VHE systems */
__kvm_vcpu_run(struct kvm_vcpu * vcpu)283 int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
284 {
285 struct kvm_cpu_context *host_ctxt;
286 struct kvm_cpu_context *guest_ctxt;
287 struct kvm_s2_mmu *mmu;
288 bool pmu_switch_needed;
289 u64 exit_code;
290
291 /*
292 * Having IRQs masked via PMR when entering the guest means the GIC
293 * will not signal the CPU of interrupts of lower priority, and the
294 * only way to get out will be via guest exceptions.
295 * Naturally, we want to avoid this.
296 */
297 if (system_uses_irq_prio_masking()) {
298 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
299 pmr_sync();
300 }
301
302 host_ctxt = host_data_ptr(host_ctxt);
303 host_ctxt->__hyp_running_vcpu = vcpu;
304 guest_ctxt = &vcpu->arch.ctxt;
305
306 pmu_switch_needed = __pmu_switch_to_guest(vcpu);
307
308 __sysreg_save_state_nvhe(host_ctxt);
309 /*
310 * We must flush and disable the SPE buffer for nVHE, as
311 * the translation regime(EL1&0) is going to be loaded with
312 * that of the guest. And we must do this before we change the
313 * translation regime to EL2 (via MDCR_EL2_E2PB == 0) and
314 * before we load guest Stage1.
315 */
316 __debug_save_host_buffers_nvhe(vcpu);
317
318 /*
319 * We're about to restore some new MMU state. Make sure
320 * ongoing page-table walks that have started before we
321 * trapped to EL2 have completed. This also synchronises the
322 * above disabling of SPE and TRBE.
323 *
324 * See DDI0487I.a D8.1.5 "Out-of-context translation regimes",
325 * rule R_LFHQG and subsequent information statements.
326 */
327 dsb(nsh);
328
329 __kvm_adjust_pc(vcpu);
330
331 /*
332 * We must restore the 32-bit state before the sysregs, thanks
333 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
334 *
335 * Also, and in order to be able to deal with erratum #1319537 (A57)
336 * and #1319367 (A72), we must ensure that all VM-related sysreg are
337 * restored before we enable S2 translation.
338 */
339 __sysreg32_restore_state(vcpu);
340 __sysreg_restore_state_nvhe(guest_ctxt);
341
342 mmu = kern_hyp_va(vcpu->arch.hw_mmu);
343 __load_stage2(mmu, kern_hyp_va(mmu->arch));
344 __activate_traps(vcpu);
345
346 __hyp_vgic_restore_state(vcpu);
347 __timer_enable_traps(vcpu);
348
349 __debug_switch_to_guest(vcpu);
350
351 do {
352 /* Jump in the fire! */
353 exit_code = __guest_enter(vcpu);
354
355 /* And we're baaack! */
356 } while (fixup_guest_exit(vcpu, &exit_code));
357
358 __sysreg_save_state_nvhe(guest_ctxt);
359 __sysreg32_save_state(vcpu);
360 __timer_disable_traps(vcpu);
361 __hyp_vgic_save_state(vcpu);
362
363 /*
364 * Same thing as before the guest run: we're about to switch
365 * the MMU context, so let's make sure we don't have any
366 * ongoing EL1&0 translations.
367 */
368 dsb(nsh);
369
370 __deactivate_traps(vcpu);
371 __load_host_stage2();
372
373 __sysreg_restore_state_nvhe(host_ctxt);
374
375 if (guest_owns_fp_regs())
376 __fpsimd_save_fpexc32(vcpu);
377
378 __debug_switch_to_host(vcpu);
379 /*
380 * This must come after restoring the host sysregs, since a non-VHE
381 * system may enable SPE here and make use of the TTBRs.
382 */
383 __debug_restore_host_buffers_nvhe(vcpu);
384
385 if (pmu_switch_needed)
386 __pmu_switch_to_host(vcpu);
387
388 /* Returning to host will clear PSR.I, remask PMR if needed */
389 if (system_uses_irq_prio_masking())
390 gic_write_pmr(GIC_PRIO_IRQOFF);
391
392 host_ctxt->__hyp_running_vcpu = NULL;
393
394 return exit_code;
395 }
396
hyp_panic(void)397 asmlinkage void __noreturn hyp_panic(void)
398 {
399 u64 spsr = read_sysreg_el2(SYS_SPSR);
400 u64 elr = read_sysreg_el2(SYS_ELR);
401 u64 par = read_sysreg_par();
402 struct kvm_cpu_context *host_ctxt;
403 struct kvm_vcpu *vcpu;
404
405 host_ctxt = host_data_ptr(host_ctxt);
406 vcpu = host_ctxt->__hyp_running_vcpu;
407
408 if (vcpu) {
409 __timer_disable_traps(vcpu);
410 __deactivate_traps(vcpu);
411 __load_host_stage2();
412 __sysreg_restore_state_nvhe(host_ctxt);
413 }
414
415 /* Prepare to dump kvm nvhe hyp stacktrace */
416 kvm_nvhe_prepare_backtrace((unsigned long)__builtin_frame_address(0),
417 _THIS_IP_);
418
419 __hyp_do_panic(host_ctxt, spsr, elr, par);
420 unreachable();
421 }
422
hyp_panic_bad_stack(void)423 asmlinkage void __noreturn hyp_panic_bad_stack(void)
424 {
425 hyp_panic();
426 }
427
kvm_unexpected_el2_exception(void)428 asmlinkage void kvm_unexpected_el2_exception(void)
429 {
430 __kvm_unexpected_el2_exception();
431 }
432