1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <hyp/sysreg-sr.h>
8
9 #include <linux/compiler.h>
10 #include <linux/kvm_host.h>
11
12 #include <asm/kprobes.h>
13 #include <asm/kvm_asm.h>
14 #include <asm/kvm_emulate.h>
15 #include <asm/kvm_hyp.h>
16 #include <asm/kvm_nested.h>
17
__sysreg_save_vel2_state(struct kvm_vcpu * vcpu)18 static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
19 {
20 /* These registers are common with EL1 */
21 __vcpu_assign_sys_reg(vcpu, PAR_EL1, read_sysreg(par_el1));
22 __vcpu_assign_sys_reg(vcpu, TPIDR_EL1, read_sysreg(tpidr_el1));
23
24 __vcpu_assign_sys_reg(vcpu, ESR_EL2, read_sysreg_el1(SYS_ESR));
25 __vcpu_assign_sys_reg(vcpu, AFSR0_EL2, read_sysreg_el1(SYS_AFSR0));
26 __vcpu_assign_sys_reg(vcpu, AFSR1_EL2, read_sysreg_el1(SYS_AFSR1));
27 __vcpu_assign_sys_reg(vcpu, FAR_EL2, read_sysreg_el1(SYS_FAR));
28 __vcpu_assign_sys_reg(vcpu, MAIR_EL2, read_sysreg_el1(SYS_MAIR));
29 __vcpu_assign_sys_reg(vcpu, VBAR_EL2, read_sysreg_el1(SYS_VBAR));
30 __vcpu_assign_sys_reg(vcpu, CONTEXTIDR_EL2, read_sysreg_el1(SYS_CONTEXTIDR));
31 __vcpu_assign_sys_reg(vcpu, AMAIR_EL2, read_sysreg_el1(SYS_AMAIR));
32
33 /*
34 * In VHE mode those registers are compatible between EL1 and EL2,
35 * and the guest uses the _EL1 versions on the CPU naturally.
36 * So we save them into their _EL2 versions here.
37 * For nVHE mode we trap accesses to those registers, so our
38 * _EL2 copy in sys_regs[] is always up-to-date and we don't need
39 * to save anything here.
40 */
41 if (vcpu_el2_e2h_is_set(vcpu)) {
42 u64 val;
43
44 /*
45 * We don't save CPTR_EL2, as accesses to CPACR_EL1
46 * are always trapped, ensuring that the in-memory
47 * copy is always up-to-date. A small blessing...
48 */
49 __vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR));
50 __vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0));
51 __vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1));
52 __vcpu_assign_sys_reg(vcpu, TCR_EL2, read_sysreg_el1(SYS_TCR));
53
54 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) {
55 __vcpu_assign_sys_reg(vcpu, TCR2_EL2, read_sysreg_el1(SYS_TCR2));
56
57 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) {
58 __vcpu_assign_sys_reg(vcpu, PIRE0_EL2, read_sysreg_el1(SYS_PIRE0));
59 __vcpu_assign_sys_reg(vcpu, PIR_EL2, read_sysreg_el1(SYS_PIR));
60 }
61
62 if (ctxt_has_s1poe(&vcpu->arch.ctxt))
63 __vcpu_assign_sys_reg(vcpu, POR_EL2, read_sysreg_el1(SYS_POR));
64 }
65
66 /*
67 * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where
68 * the interesting CNTHCTL_EL2 bits live. So preserve these
69 * bits when reading back the guest-visible value.
70 */
71 val = read_sysreg_el1(SYS_CNTKCTL);
72 val &= CNTKCTL_VALID_BITS;
73 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS);
74 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val);
75 }
76
77 __vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));
78 __vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR));
79 __vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR));
80 }
81
__sysreg_restore_vel2_state(struct kvm_vcpu * vcpu)82 static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
83 {
84 u64 val;
85
86 /* These registers are common with EL1 */
87 write_sysreg(__vcpu_sys_reg(vcpu, PAR_EL1), par_el1);
88 write_sysreg(__vcpu_sys_reg(vcpu, TPIDR_EL1), tpidr_el1);
89
90 write_sysreg(ctxt_midr_el1(&vcpu->arch.ctxt), vpidr_el2);
91 write_sysreg(__vcpu_sys_reg(vcpu, MPIDR_EL1), vmpidr_el2);
92 write_sysreg_el1(__vcpu_sys_reg(vcpu, MAIR_EL2), SYS_MAIR);
93 write_sysreg_el1(__vcpu_sys_reg(vcpu, VBAR_EL2), SYS_VBAR);
94 write_sysreg_el1(__vcpu_sys_reg(vcpu, CONTEXTIDR_EL2), SYS_CONTEXTIDR);
95 write_sysreg_el1(__vcpu_sys_reg(vcpu, AMAIR_EL2), SYS_AMAIR);
96
97 if (vcpu_el2_e2h_is_set(vcpu)) {
98 /*
99 * In VHE mode those registers are compatible between
100 * EL1 and EL2.
101 */
102 write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2), SYS_SCTLR);
103 write_sysreg_el1(__vcpu_sys_reg(vcpu, CPTR_EL2), SYS_CPACR);
104 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2), SYS_TTBR0);
105 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR1_EL2), SYS_TTBR1);
106 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR_EL2), SYS_TCR);
107 write_sysreg_el1(__vcpu_sys_reg(vcpu, CNTHCTL_EL2), SYS_CNTKCTL);
108 } else {
109 /*
110 * CNTHCTL_EL2 only affects EL1 when running nVHE, so
111 * no need to restore it.
112 */
113 val = translate_sctlr_el2_to_sctlr_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2));
114 write_sysreg_el1(val, SYS_SCTLR);
115 val = translate_cptr_el2_to_cpacr_el1(__vcpu_sys_reg(vcpu, CPTR_EL2));
116 write_sysreg_el1(val, SYS_CPACR);
117 val = translate_ttbr0_el2_to_ttbr0_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2));
118 write_sysreg_el1(val, SYS_TTBR0);
119 val = translate_tcr_el2_to_tcr_el1(__vcpu_sys_reg(vcpu, TCR_EL2));
120 write_sysreg_el1(val, SYS_TCR);
121 }
122
123 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) {
124 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR2_EL2), SYS_TCR2);
125
126 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) {
127 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIR_EL2), SYS_PIR);
128 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIRE0_EL2), SYS_PIRE0);
129 }
130
131 if (ctxt_has_s1poe(&vcpu->arch.ctxt))
132 write_sysreg_el1(__vcpu_sys_reg(vcpu, POR_EL2), SYS_POR);
133 }
134
135 write_sysreg_el1(__vcpu_sys_reg(vcpu, ESR_EL2), SYS_ESR);
136 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR0_EL2), SYS_AFSR0);
137 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR1_EL2), SYS_AFSR1);
138 write_sysreg_el1(__vcpu_sys_reg(vcpu, FAR_EL2), SYS_FAR);
139 write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
140 write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
141 write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
142 }
143
144 /*
145 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
146 * pstate, which are handled as part of the el2 return state) on every
147 * switch (sp_el0 is being dealt with in the assembly code).
148 * tpidr_el0 and tpidrro_el0 only need to be switched when going
149 * to host userspace or a different VCPU. EL1 registers only need to be
150 * switched when potentially going to run a different VCPU. The latter two
151 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
152 */
153
sysreg_save_host_state_vhe(struct kvm_cpu_context * ctxt)154 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
155 {
156 __sysreg_save_common_state(ctxt);
157 }
158 NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
159
sysreg_save_guest_state_vhe(struct kvm_cpu_context * ctxt)160 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
161 {
162 __sysreg_save_common_state(ctxt);
163 __sysreg_save_el2_return_state(ctxt);
164 }
165 NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
166
sysreg_restore_host_state_vhe(struct kvm_cpu_context * ctxt)167 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
168 {
169 __sysreg_restore_common_state(ctxt);
170 }
171 NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
172
sysreg_restore_guest_state_vhe(struct kvm_cpu_context * ctxt)173 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
174 {
175 __sysreg_restore_common_state(ctxt);
176 __sysreg_restore_el2_return_state(ctxt);
177 }
178 NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
179
180 /**
181 * __vcpu_load_switch_sysregs - Load guest system registers to the physical CPU
182 *
183 * @vcpu: The VCPU pointer
184 *
185 * Load system registers that do not affect the host's execution, for
186 * example EL1 system registers on a VHE system where the host kernel
187 * runs at EL2. This function is called from KVM's vcpu_load() function
188 * and loading system register state early avoids having to load them on
189 * every entry to the VM.
190 */
__vcpu_load_switch_sysregs(struct kvm_vcpu * vcpu)191 void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu)
192 {
193 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
194 struct kvm_cpu_context *host_ctxt;
195 u64 midr, mpidr;
196
197 host_ctxt = host_data_ptr(host_ctxt);
198 __sysreg_save_user_state(host_ctxt);
199
200 /*
201 * When running a normal EL1 guest, we only load a new vcpu
202 * after a context switch, which imvolves a DSB, so all
203 * speculative EL1&0 walks will have already completed.
204 * If running NV, the vcpu may transition between vEL1 and
205 * vEL2 without a context switch, so make sure we complete
206 * those walks before loading a new context.
207 */
208 if (vcpu_has_nv(vcpu))
209 dsb(nsh);
210
211 /*
212 * Load guest EL1 and user state
213 *
214 * We must restore the 32-bit state before the sysregs, thanks
215 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
216 */
217 __sysreg32_restore_state(vcpu);
218 __sysreg_restore_user_state(guest_ctxt);
219
220 if (unlikely(is_hyp_ctxt(vcpu))) {
221 __sysreg_restore_vel2_state(vcpu);
222 } else {
223 if (vcpu_has_nv(vcpu)) {
224 /*
225 * As we're restoring a nested guest, set the value
226 * provided by the guest hypervisor.
227 */
228 midr = ctxt_sys_reg(guest_ctxt, VPIDR_EL2);
229 mpidr = ctxt_sys_reg(guest_ctxt, VMPIDR_EL2);
230 } else {
231 midr = ctxt_midr_el1(guest_ctxt);
232 mpidr = ctxt_sys_reg(guest_ctxt, MPIDR_EL1);
233 }
234
235 __sysreg_restore_el1_state(guest_ctxt, midr, mpidr);
236 }
237
238 vcpu_set_flag(vcpu, SYSREGS_ON_CPU);
239 }
240
241 /**
242 * __vcpu_put_switch_sysregs - Restore host system registers to the physical CPU
243 *
244 * @vcpu: The VCPU pointer
245 *
246 * Save guest system registers that do not affect the host's execution, for
247 * example EL1 system registers on a VHE system where the host kernel
248 * runs at EL2. This function is called from KVM's vcpu_put() function
249 * and deferring saving system register state until we're no longer running the
250 * VCPU avoids having to save them on every exit from the VM.
251 */
__vcpu_put_switch_sysregs(struct kvm_vcpu * vcpu)252 void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu)
253 {
254 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
255 struct kvm_cpu_context *host_ctxt;
256
257 host_ctxt = host_data_ptr(host_ctxt);
258
259 if (unlikely(is_hyp_ctxt(vcpu)))
260 __sysreg_save_vel2_state(vcpu);
261 else
262 __sysreg_save_el1_state(guest_ctxt);
263
264 __sysreg_save_user_state(guest_ctxt);
265 __sysreg32_save_state(vcpu);
266
267 /* Restore host user state */
268 __sysreg_restore_user_state(host_ctxt);
269
270 vcpu_clear_flag(vcpu, SYSREGS_ON_CPU);
271 }
272