1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/sysreg-sr.h> 8 9 #include <linux/compiler.h> 10 #include <linux/kvm_host.h> 11 12 #include <asm/kprobes.h> 13 #include <asm/kvm_asm.h> 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_hyp.h> 16 #include <asm/kvm_nested.h> 17 18 static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) 19 { 20 /* These registers are common with EL1 */ 21 __vcpu_sys_reg(vcpu, PAR_EL1) = read_sysreg(par_el1); 22 __vcpu_sys_reg(vcpu, TPIDR_EL1) = read_sysreg(tpidr_el1); 23 24 __vcpu_sys_reg(vcpu, ESR_EL2) = read_sysreg_el1(SYS_ESR); 25 __vcpu_sys_reg(vcpu, AFSR0_EL2) = read_sysreg_el1(SYS_AFSR0); 26 __vcpu_sys_reg(vcpu, AFSR1_EL2) = read_sysreg_el1(SYS_AFSR1); 27 __vcpu_sys_reg(vcpu, FAR_EL2) = read_sysreg_el1(SYS_FAR); 28 __vcpu_sys_reg(vcpu, MAIR_EL2) = read_sysreg_el1(SYS_MAIR); 29 __vcpu_sys_reg(vcpu, VBAR_EL2) = read_sysreg_el1(SYS_VBAR); 30 __vcpu_sys_reg(vcpu, CONTEXTIDR_EL2) = read_sysreg_el1(SYS_CONTEXTIDR); 31 __vcpu_sys_reg(vcpu, AMAIR_EL2) = read_sysreg_el1(SYS_AMAIR); 32 33 /* 34 * In VHE mode those registers are compatible between EL1 and EL2, 35 * and the guest uses the _EL1 versions on the CPU naturally. 36 * So we save them into their _EL2 versions here. 37 * For nVHE mode we trap accesses to those registers, so our 38 * _EL2 copy in sys_regs[] is always up-to-date and we don't need 39 * to save anything here. 40 */ 41 if (vcpu_el2_e2h_is_set(vcpu)) { 42 u64 val; 43 44 /* 45 * We don't save CPTR_EL2, as accesses to CPACR_EL1 46 * are always trapped, ensuring that the in-memory 47 * copy is always up-to-date. A small blessing... 48 */ 49 __vcpu_sys_reg(vcpu, SCTLR_EL2) = read_sysreg_el1(SYS_SCTLR); 50 __vcpu_sys_reg(vcpu, TTBR0_EL2) = read_sysreg_el1(SYS_TTBR0); 51 __vcpu_sys_reg(vcpu, TTBR1_EL2) = read_sysreg_el1(SYS_TTBR1); 52 __vcpu_sys_reg(vcpu, TCR_EL2) = read_sysreg_el1(SYS_TCR); 53 54 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) 55 __vcpu_sys_reg(vcpu, TCR2_EL2) = read_sysreg_el1(SYS_TCR2); 56 57 /* 58 * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where 59 * the interesting CNTHCTL_EL2 bits live. So preserve these 60 * bits when reading back the guest-visible value. 61 */ 62 val = read_sysreg_el1(SYS_CNTKCTL); 63 val &= CNTKCTL_VALID_BITS; 64 __vcpu_sys_reg(vcpu, CNTHCTL_EL2) &= ~CNTKCTL_VALID_BITS; 65 __vcpu_sys_reg(vcpu, CNTHCTL_EL2) |= val; 66 } 67 68 __vcpu_sys_reg(vcpu, SP_EL2) = read_sysreg(sp_el1); 69 __vcpu_sys_reg(vcpu, ELR_EL2) = read_sysreg_el1(SYS_ELR); 70 __vcpu_sys_reg(vcpu, SPSR_EL2) = read_sysreg_el1(SYS_SPSR); 71 } 72 73 static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu) 74 { 75 u64 val; 76 77 /* These registers are common with EL1 */ 78 write_sysreg(__vcpu_sys_reg(vcpu, PAR_EL1), par_el1); 79 write_sysreg(__vcpu_sys_reg(vcpu, TPIDR_EL1), tpidr_el1); 80 81 write_sysreg(__vcpu_sys_reg(vcpu, MPIDR_EL1), vmpidr_el2); 82 write_sysreg_el1(__vcpu_sys_reg(vcpu, MAIR_EL2), SYS_MAIR); 83 write_sysreg_el1(__vcpu_sys_reg(vcpu, VBAR_EL2), SYS_VBAR); 84 write_sysreg_el1(__vcpu_sys_reg(vcpu, CONTEXTIDR_EL2), SYS_CONTEXTIDR); 85 write_sysreg_el1(__vcpu_sys_reg(vcpu, AMAIR_EL2), SYS_AMAIR); 86 87 if (vcpu_el2_e2h_is_set(vcpu)) { 88 /* 89 * In VHE mode those registers are compatible between 90 * EL1 and EL2. 91 */ 92 write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2), SYS_SCTLR); 93 write_sysreg_el1(__vcpu_sys_reg(vcpu, CPTR_EL2), SYS_CPACR); 94 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2), SYS_TTBR0); 95 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR1_EL2), SYS_TTBR1); 96 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR_EL2), SYS_TCR); 97 write_sysreg_el1(__vcpu_sys_reg(vcpu, CNTHCTL_EL2), SYS_CNTKCTL); 98 } else { 99 /* 100 * CNTHCTL_EL2 only affects EL1 when running nVHE, so 101 * no need to restore it. 102 */ 103 val = translate_sctlr_el2_to_sctlr_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2)); 104 write_sysreg_el1(val, SYS_SCTLR); 105 val = translate_cptr_el2_to_cpacr_el1(__vcpu_sys_reg(vcpu, CPTR_EL2)); 106 write_sysreg_el1(val, SYS_CPACR); 107 val = translate_ttbr0_el2_to_ttbr0_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2)); 108 write_sysreg_el1(val, SYS_TTBR0); 109 val = translate_tcr_el2_to_tcr_el1(__vcpu_sys_reg(vcpu, TCR_EL2)); 110 write_sysreg_el1(val, SYS_TCR); 111 } 112 113 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) 114 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR2_EL2), SYS_TCR2); 115 116 117 write_sysreg_el1(__vcpu_sys_reg(vcpu, ESR_EL2), SYS_ESR); 118 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR0_EL2), SYS_AFSR0); 119 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR1_EL2), SYS_AFSR1); 120 write_sysreg_el1(__vcpu_sys_reg(vcpu, FAR_EL2), SYS_FAR); 121 write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1); 122 write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR); 123 write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR); 124 } 125 126 /* 127 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and 128 * pstate, which are handled as part of the el2 return state) on every 129 * switch (sp_el0 is being dealt with in the assembly code). 130 * tpidr_el0 and tpidrro_el0 only need to be switched when going 131 * to host userspace or a different VCPU. EL1 registers only need to be 132 * switched when potentially going to run a different VCPU. The latter two 133 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put. 134 */ 135 136 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt) 137 { 138 __sysreg_save_common_state(ctxt); 139 } 140 NOKPROBE_SYMBOL(sysreg_save_host_state_vhe); 141 142 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt) 143 { 144 __sysreg_save_common_state(ctxt); 145 __sysreg_save_el2_return_state(ctxt); 146 } 147 NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe); 148 149 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt) 150 { 151 __sysreg_restore_common_state(ctxt); 152 } 153 NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe); 154 155 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt) 156 { 157 __sysreg_restore_common_state(ctxt); 158 __sysreg_restore_el2_return_state(ctxt); 159 } 160 NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe); 161 162 /** 163 * __vcpu_load_switch_sysregs - Load guest system registers to the physical CPU 164 * 165 * @vcpu: The VCPU pointer 166 * 167 * Load system registers that do not affect the host's execution, for 168 * example EL1 system registers on a VHE system where the host kernel 169 * runs at EL2. This function is called from KVM's vcpu_load() function 170 * and loading system register state early avoids having to load them on 171 * every entry to the VM. 172 */ 173 void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu) 174 { 175 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 176 struct kvm_cpu_context *host_ctxt; 177 u64 mpidr; 178 179 host_ctxt = host_data_ptr(host_ctxt); 180 __sysreg_save_user_state(host_ctxt); 181 182 /* 183 * When running a normal EL1 guest, we only load a new vcpu 184 * after a context switch, which imvolves a DSB, so all 185 * speculative EL1&0 walks will have already completed. 186 * If running NV, the vcpu may transition between vEL1 and 187 * vEL2 without a context switch, so make sure we complete 188 * those walks before loading a new context. 189 */ 190 if (vcpu_has_nv(vcpu)) 191 dsb(nsh); 192 193 /* 194 * Load guest EL1 and user state 195 * 196 * We must restore the 32-bit state before the sysregs, thanks 197 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). 198 */ 199 __sysreg32_restore_state(vcpu); 200 __sysreg_restore_user_state(guest_ctxt); 201 202 if (unlikely(__is_hyp_ctxt(guest_ctxt))) { 203 __sysreg_restore_vel2_state(vcpu); 204 } else { 205 if (vcpu_has_nv(vcpu)) { 206 /* 207 * Use the guest hypervisor's VPIDR_EL2 when in a 208 * nested state. The hardware value of MIDR_EL1 gets 209 * restored on put. 210 */ 211 write_sysreg(ctxt_sys_reg(guest_ctxt, VPIDR_EL2), vpidr_el2); 212 213 /* 214 * As we're restoring a nested guest, set the value 215 * provided by the guest hypervisor. 216 */ 217 mpidr = ctxt_sys_reg(guest_ctxt, VMPIDR_EL2); 218 } else { 219 mpidr = ctxt_sys_reg(guest_ctxt, MPIDR_EL1); 220 } 221 222 __sysreg_restore_el1_state(guest_ctxt, mpidr); 223 } 224 225 vcpu_set_flag(vcpu, SYSREGS_ON_CPU); 226 } 227 228 /** 229 * __vcpu_put_switch_sysregs - Restore host system registers to the physical CPU 230 * 231 * @vcpu: The VCPU pointer 232 * 233 * Save guest system registers that do not affect the host's execution, for 234 * example EL1 system registers on a VHE system where the host kernel 235 * runs at EL2. This function is called from KVM's vcpu_put() function 236 * and deferring saving system register state until we're no longer running the 237 * VCPU avoids having to save them on every exit from the VM. 238 */ 239 void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu) 240 { 241 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; 242 struct kvm_cpu_context *host_ctxt; 243 244 host_ctxt = host_data_ptr(host_ctxt); 245 246 if (unlikely(__is_hyp_ctxt(guest_ctxt))) 247 __sysreg_save_vel2_state(vcpu); 248 else 249 __sysreg_save_el1_state(guest_ctxt); 250 251 __sysreg_save_user_state(guest_ctxt); 252 __sysreg32_save_state(vcpu); 253 254 /* Restore host user state */ 255 __sysreg_restore_user_state(host_ctxt); 256 257 /* If leaving a nesting guest, restore MIDR_EL1 default view */ 258 if (vcpu_has_nv(vcpu)) 259 write_sysreg(read_cpuid_id(), vpidr_el2); 260 261 vcpu_clear_flag(vcpu, SYSREGS_ON_CPU); 262 } 263