1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <asm/kvm_hyp.h> 8 #include <asm/kvm_mmu.h> 9 #include <asm/tlbflush.h> 10 11 #include <nvhe/mem_protect.h> 12 13 struct tlb_inv_context { 14 struct kvm_s2_mmu *mmu; 15 u64 tcr; 16 u64 sctlr; 17 }; 18 19 static void enter_vmid_context(struct kvm_s2_mmu *mmu, 20 struct tlb_inv_context *cxt, 21 bool nsh) 22 { 23 struct kvm_s2_mmu *host_s2_mmu = &host_mmu.arch.mmu; 24 struct kvm_cpu_context *host_ctxt; 25 struct kvm_vcpu *vcpu; 26 27 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 28 vcpu = host_ctxt->__hyp_running_vcpu; 29 cxt->mmu = NULL; 30 31 /* 32 * We have two requirements: 33 * 34 * - ensure that the page table updates are visible to all 35 * CPUs, for which a dsb(DOMAIN-st) is what we need, DOMAIN 36 * being either ish or nsh, depending on the invalidation 37 * type. 38 * 39 * - complete any speculative page table walk started before 40 * we trapped to EL2 so that we can mess with the MM 41 * registers out of context, for which dsb(nsh) is enough 42 * 43 * The composition of these two barriers is a dsb(DOMAIN), and 44 * the 'nsh' parameter tracks the distinction between 45 * Inner-Shareable and Non-Shareable, as specified by the 46 * callers. 47 */ 48 if (nsh) 49 dsb(nsh); 50 else 51 dsb(ish); 52 53 /* 54 * If we're already in the desired context, then there's nothing to do. 55 */ 56 if (vcpu) { 57 /* 58 * We're in guest context. However, for this to work, this needs 59 * to be called from within __kvm_vcpu_run(), which ensures that 60 * __hyp_running_vcpu is set to the current guest vcpu. 61 */ 62 if (mmu == vcpu->arch.hw_mmu || WARN_ON(mmu != host_s2_mmu)) 63 return; 64 65 cxt->mmu = vcpu->arch.hw_mmu; 66 } else { 67 /* We're in host context. */ 68 if (mmu == host_s2_mmu) 69 return; 70 71 cxt->mmu = host_s2_mmu; 72 } 73 74 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 75 u64 val; 76 77 /* 78 * For CPUs that are affected by ARM 1319367, we need to 79 * avoid a Stage-1 walk with the old VMID while we have 80 * the new VMID set in the VTTBR in order to invalidate TLBs. 81 * We're guaranteed that the host S1 MMU is enabled, so 82 * we can simply set the EPD bits to avoid any further 83 * TLB fill. For guests, we ensure that the S1 MMU is 84 * temporarily enabled in the next context. 85 */ 86 val = cxt->tcr = read_sysreg_el1(SYS_TCR); 87 val |= TCR_EPD1_MASK | TCR_EPD0_MASK; 88 write_sysreg_el1(val, SYS_TCR); 89 isb(); 90 91 if (vcpu) { 92 val = cxt->sctlr = read_sysreg_el1(SYS_SCTLR); 93 if (!(val & SCTLR_ELx_M)) { 94 val |= SCTLR_ELx_M; 95 write_sysreg_el1(val, SYS_SCTLR); 96 isb(); 97 } 98 } else { 99 /* The host S1 MMU is always enabled. */ 100 cxt->sctlr = SCTLR_ELx_M; 101 } 102 } 103 104 /* 105 * __load_stage2() includes an ISB only when the AT 106 * workaround is applied. Take care of the opposite condition, 107 * ensuring that we always have an ISB, but not two ISBs back 108 * to back. 109 */ 110 if (vcpu) 111 __load_host_stage2(); 112 else 113 __load_stage2(mmu, kern_hyp_va(mmu->arch)); 114 115 asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT)); 116 } 117 118 static void exit_vmid_context(struct tlb_inv_context *cxt) 119 { 120 struct kvm_s2_mmu *mmu = cxt->mmu; 121 struct kvm_cpu_context *host_ctxt; 122 struct kvm_vcpu *vcpu; 123 124 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 125 vcpu = host_ctxt->__hyp_running_vcpu; 126 127 if (!mmu) 128 return; 129 130 if (vcpu) 131 __load_stage2(mmu, kern_hyp_va(mmu->arch)); 132 else 133 __load_host_stage2(); 134 135 /* Ensure write of the old VMID */ 136 isb(); 137 138 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { 139 if (!(cxt->sctlr & SCTLR_ELx_M)) { 140 write_sysreg_el1(cxt->sctlr, SYS_SCTLR); 141 isb(); 142 } 143 144 write_sysreg_el1(cxt->tcr, SYS_TCR); 145 } 146 } 147 148 void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, 149 phys_addr_t ipa, int level) 150 { 151 struct tlb_inv_context cxt; 152 153 /* Switch to requested VMID */ 154 enter_vmid_context(mmu, &cxt, false); 155 156 /* 157 * We could do so much better if we had the VA as well. 158 * Instead, we invalidate Stage-2 for this IPA, and the 159 * whole of Stage-1. Weep... 160 */ 161 ipa >>= 12; 162 __tlbi_level(ipas2e1is, ipa, level); 163 164 /* 165 * We have to ensure completion of the invalidation at Stage-2, 166 * since a table walk on another CPU could refill a TLB with a 167 * complete (S1 + S2) walk based on the old Stage-2 mapping if 168 * the Stage-1 invalidation happened first. 169 */ 170 dsb(ish); 171 __tlbi(vmalle1is); 172 dsb(ish); 173 isb(); 174 175 exit_vmid_context(&cxt); 176 } 177 178 void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu, 179 phys_addr_t ipa, int level) 180 { 181 struct tlb_inv_context cxt; 182 183 /* Switch to requested VMID */ 184 enter_vmid_context(mmu, &cxt, true); 185 186 /* 187 * We could do so much better if we had the VA as well. 188 * Instead, we invalidate Stage-2 for this IPA, and the 189 * whole of Stage-1. Weep... 190 */ 191 ipa >>= 12; 192 __tlbi_level(ipas2e1, ipa, level); 193 194 /* 195 * We have to ensure completion of the invalidation at Stage-2, 196 * since a table walk on another CPU could refill a TLB with a 197 * complete (S1 + S2) walk based on the old Stage-2 mapping if 198 * the Stage-1 invalidation happened first. 199 */ 200 dsb(nsh); 201 __tlbi(vmalle1); 202 dsb(nsh); 203 isb(); 204 205 exit_vmid_context(&cxt); 206 } 207 208 void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, 209 phys_addr_t start, unsigned long pages) 210 { 211 struct tlb_inv_context cxt; 212 unsigned long stride; 213 214 /* 215 * Since the range of addresses may not be mapped at 216 * the same level, assume the worst case as PAGE_SIZE 217 */ 218 stride = PAGE_SIZE; 219 start = round_down(start, stride); 220 221 /* Switch to requested VMID */ 222 enter_vmid_context(mmu, &cxt, false); 223 224 __flush_s2_tlb_range_op(ipas2e1is, start, pages, stride, 225 TLBI_TTL_UNKNOWN); 226 227 dsb(ish); 228 __tlbi(vmalle1is); 229 dsb(ish); 230 isb(); 231 232 exit_vmid_context(&cxt); 233 } 234 235 void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) 236 { 237 struct tlb_inv_context cxt; 238 239 /* Switch to requested VMID */ 240 enter_vmid_context(mmu, &cxt, false); 241 242 __tlbi(vmalls12e1is); 243 dsb(ish); 244 isb(); 245 246 exit_vmid_context(&cxt); 247 } 248 249 void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu) 250 { 251 struct tlb_inv_context cxt; 252 253 /* Switch to requested VMID */ 254 enter_vmid_context(mmu, &cxt, false); 255 256 __tlbi(vmalle1); 257 asm volatile("ic iallu"); 258 dsb(nsh); 259 isb(); 260 261 exit_vmid_context(&cxt); 262 } 263 264 void __kvm_flush_vm_context(void) 265 { 266 /* Same remark as in enter_vmid_context() */ 267 dsb(ish); 268 __tlbi(alle1is); 269 dsb(ish); 270 } 271