1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/sysreg-sr.h> 8 9 #include <linux/compiler.h> 10 #include <linux/kvm_host.h> 11 12 #include <asm/kprobes.h> 13 #include <asm/kvm_asm.h> 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_hyp.h> 16 #include <asm/kvm_nested.h> 17 18 static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) 19 { 20 /* These registers are common with EL1 */ 21 __vcpu_assign_sys_reg(vcpu, PAR_EL1, read_sysreg(par_el1)); 22 __vcpu_assign_sys_reg(vcpu, TPIDR_EL1, read_sysreg(tpidr_el1)); 23 24 __vcpu_assign_sys_reg(vcpu, ESR_EL2, read_sysreg_el1(SYS_ESR)); 25 __vcpu_assign_sys_reg(vcpu, AFSR0_EL2, read_sysreg_el1(SYS_AFSR0)); 26 __vcpu_assign_sys_reg(vcpu, AFSR1_EL2, read_sysreg_el1(SYS_AFSR1)); 27 __vcpu_assign_sys_reg(vcpu, FAR_EL2, read_sysreg_el1(SYS_FAR)); 28 __vcpu_assign_sys_reg(vcpu, MAIR_EL2, read_sysreg_el1(SYS_MAIR)); 29 __vcpu_assign_sys_reg(vcpu, VBAR_EL2, read_sysreg_el1(SYS_VBAR)); 30 __vcpu_assign_sys_reg(vcpu, CONTEXTIDR_EL2, read_sysreg_el1(SYS_CONTEXTIDR)); 31 __vcpu_assign_sys_reg(vcpu, AMAIR_EL2, read_sysreg_el1(SYS_AMAIR)); 32 33 /* 34 * In VHE mode those registers are compatible between EL1 and EL2, 35 * and the guest uses the _EL1 versions on the CPU naturally. 36 * So we save them into their _EL2 versions here. 37 * For nVHE mode we trap accesses to those registers, so our 38 * _EL2 copy in sys_regs[] is always up-to-date and we don't need 39 * to save anything here. 40 */ 41 if (vcpu_el2_e2h_is_set(vcpu)) { 42 u64 val; 43 44 /* 45 * We don't save CPTR_EL2, as accesses to CPACR_EL1 46 * are always trapped, ensuring that the in-memory 47 * copy is always up-to-date. A small blessing... 48 */ 49 __vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR)); 50 __vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0)); 51 __vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1)); 52 __vcpu_assign_sys_reg(vcpu, TCR_EL2, read_sysreg_el1(SYS_TCR)); 53 54 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) { 55 __vcpu_assign_sys_reg(vcpu, TCR2_EL2, read_sysreg_el1(SYS_TCR2)); 56 57 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) { 58 __vcpu_assign_sys_reg(vcpu, PIRE0_EL2, read_sysreg_el1(SYS_PIRE0)); 59 __vcpu_assign_sys_reg(vcpu, PIR_EL2, read_sysreg_el1(SYS_PIR)); 60 } 61 62 if (ctxt_has_s1poe(&vcpu->arch.ctxt)) 63 __vcpu_assign_sys_reg(vcpu, POR_EL2, read_sysreg_el1(SYS_POR)); 64 } 65 66 /* 67 * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where 68 * the interesting CNTHCTL_EL2 bits live. So preserve these 69 * bits when reading back the guest-visible value. 70 */ 71 val = read_sysreg_el1(SYS_CNTKCTL); 72 val &= CNTKCTL_VALID_BITS; 73 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS); 74 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val); 75 } 76 77 __vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1)); 78 __vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR)); 79 __vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR)); 80 } 81 82 static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu) 83 { 84 u64 val; 85 86 /* These registers are common with EL1 */ 87 write_sysreg(__vcpu_sys_reg(vcpu, PAR_EL1), par_el1); 88 write_sysreg(__vcpu_sys_reg(vcpu, TPIDR_EL1), tpidr_el1); 89 90 write_sysreg(ctxt_midr_el1(&vcpu->arch.ctxt), vpidr_el2); 91 write_sysreg(__vcpu_sys_reg(vcpu, MPIDR_EL1), vmpidr_el2); 92 write_sysreg_el1(__vcpu_sys_reg(vcpu, MAIR_EL2), SYS_MAIR); 93 write_sysreg_el1(__vcpu_sys_reg(vcpu, VBAR_EL2), SYS_VBAR); 94 write_sysreg_el1(__vcpu_sys_reg(vcpu, CONTEXTIDR_EL2), SYS_CONTEXTIDR); 95 write_sysreg_el1(__vcpu_sys_reg(vcpu, AMAIR_EL2), SYS_AMAIR); 96 97 if (vcpu_el2_e2h_is_set(vcpu)) { 98 /* 99 * In VHE mode those registers are compatible between 100 * EL1 and EL2. 101 */ 102 write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2), SYS_SCTLR); 103 write_sysreg_el1(__vcpu_sys_reg(vcpu, CPTR_EL2), SYS_CPACR); 104 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2), SYS_TTBR0); 105 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR1_EL2), SYS_TTBR1); 106 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR_EL2), SYS_TCR); 107 write_sysreg_el1(__vcpu_sys_reg(vcpu, CNTHCTL_EL2), SYS_CNTKCTL); 108 } else { 109 /* 110 * CNTHCTL_EL2 only affects EL1 when running nVHE, so 111 * no need to restore it. 112 */ 113 val = translate_sctlr_el2_to_sctlr_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2)); 114 write_sysreg_el1(val, SYS_SCTLR); 115 val = translate_cptr_el2_to_cpacr_el1(__vcpu_sys_reg(vcpu, CPTR_EL2)); 116 write_sysreg_el1(val, SYS_CPACR); 117 val = translate_ttbr0_el2_to_ttbr0_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2)); 118 write_sysreg_el1(val, SYS_TTBR0); 119 val = translate_tcr_el2_to_tcr_el1(__vcpu_sys_reg(vcpu, TCR_EL2)); 120 write_sysreg_el1(val, SYS_TCR); 121 } 122 123 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) { 124 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR2_EL2), SYS_TCR2); 125 126 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) { 127 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIR_EL2), SYS_PIR); 128 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIRE0_EL2), SYS_PIRE0); 129 } 130 131 if (ctxt_has_s1poe(&vcpu->arch.ctxt)) 132 write_sysreg_el1(__vcpu_sys_reg(vcpu, POR_EL2), SYS_POR); 133 } 134 135 write_sysreg_el1(__vcpu_sys_reg(vcpu, ESR_EL2), SYS_ESR); 136 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR0_EL2), SYS_AFSR0); 137 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR1_EL2), SYS_AFSR1); 138 write_sysreg_el1(__vcpu_sys_reg(vcpu, FAR_EL2), SYS_FAR); 139 write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1); 140 write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR); 141 write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR); 142 } 143 144 /* 145 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and 146 * pstate, which are handled as part of the el2 return state) on every 147 * switch (sp_el0 is being dealt with in the assembly code). 148 * tpidr_el0 and tpidrro_el0 only need to be switched when going 149 * to host userspace or a different VCPU. EL1 registers only need to be 150 * switched when potentially going to run a different VCPU. The latter two 151 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put. 152 */ 153 154 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt) 155 { 156 __sysreg_save_common_state(ctxt); 157 } 158 NOKPROBE_SYMBOL(sysreg_save_host_state_vhe); 159 160 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt) 161 { 162 __sysreg_save_common_state(ctxt); 163 __sysreg_save_el2_return_state(ctxt); 164 } 165 NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe); 166 167 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt) 168 { 169 __sysreg_restore_common_state(ctxt); 170 } 171 NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe); 172 173 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt) 174 { 175 __sysreg_restore_common_state(ctxt); 176 __sysreg_restore_el2_return_state(ctxt); 177 } 178 NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe); 179 180 /** 181 * __vcpu_load_switch_sysregs - Load guest system registers to the physical CPU 182 * 183 * @vcpu: The VCPU pointer 184 * 185 * Load system registers that do not affect the host's execution, for 186 * example EL1 system registers on a VHE system where the host kernel 187 * runs at EL2. This function is called from KVM's vcpu_load() function 188 * and loading system register state early avoids having to load them on 189 * every entry to the VM. 190 */ 191 void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu) 192 { 193 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 194 struct kvm_cpu_context *host_ctxt; 195 u64 midr, mpidr; 196 197 host_ctxt = host_data_ptr(host_ctxt); 198 __sysreg_save_user_state(host_ctxt); 199 200 /* 201 * When running a normal EL1 guest, we only load a new vcpu 202 * after a context switch, which imvolves a DSB, so all 203 * speculative EL1&0 walks will have already completed. 204 * If running NV, the vcpu may transition between vEL1 and 205 * vEL2 without a context switch, so make sure we complete 206 * those walks before loading a new context. 207 */ 208 if (vcpu_has_nv(vcpu)) 209 dsb(nsh); 210 211 /* 212 * Load guest EL1 and user state 213 * 214 * We must restore the 32-bit state before the sysregs, thanks 215 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 216 */ 217 __sysreg32_restore_state(vcpu); 218 __sysreg_restore_user_state(guest_ctxt); 219 220 if (unlikely(is_hyp_ctxt(vcpu))) { 221 __sysreg_restore_vel2_state(vcpu); 222 } else { 223 if (vcpu_has_nv(vcpu)) { 224 /* 225 * As we're restoring a nested guest, set the value 226 * provided by the guest hypervisor. 227 */ 228 midr = ctxt_sys_reg(guest_ctxt, VPIDR_EL2); 229 mpidr = ctxt_sys_reg(guest_ctxt, VMPIDR_EL2); 230 } else { 231 midr = ctxt_midr_el1(guest_ctxt); 232 mpidr = ctxt_sys_reg(guest_ctxt, MPIDR_EL1); 233 } 234 235 __sysreg_restore_el1_state(guest_ctxt, midr, mpidr); 236 } 237 238 vcpu_set_flag(vcpu, SYSREGS_ON_CPU); 239 } 240 241 /** 242 * __vcpu_put_switch_sysregs - Restore host system registers to the physical CPU 243 * 244 * @vcpu: The VCPU pointer 245 * 246 * Save guest system registers that do not affect the host's execution, for 247 * example EL1 system registers on a VHE system where the host kernel 248 * runs at EL2. This function is called from KVM's vcpu_put() function 249 * and deferring saving system register state until we're no longer running the 250 * VCPU avoids having to save them on every exit from the VM. 251 */ 252 void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu) 253 { 254 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 255 struct kvm_cpu_context *host_ctxt; 256 257 host_ctxt = host_data_ptr(host_ctxt); 258 259 if (unlikely(is_hyp_ctxt(vcpu))) 260 __sysreg_save_vel2_state(vcpu); 261 else 262 __sysreg_save_el1_state(guest_ctxt); 263 264 __sysreg_save_user_state(guest_ctxt); 265 __sysreg32_save_state(vcpu); 266 267 /* Restore host user state */ 268 __sysreg_restore_user_state(host_ctxt); 269 270 vcpu_clear_flag(vcpu, SYSREGS_ON_CPU); 271 } 272