1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022 Ventana Micro Systems Inc. 4 */ 5 6 #include <linux/bitmap.h> 7 #include <linux/cpumask.h> 8 #include <linux/errno.h> 9 #include <linux/err.h> 10 #include <linux/module.h> 11 #include <linux/smp.h> 12 #include <linux/kvm_host.h> 13 #include <asm/cacheflush.h> 14 #include <asm/csr.h> 15 #include <asm/cpufeature.h> 16 #include <asm/insn-def.h> 17 18 #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) 19 20 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, 21 gpa_t gpa, gpa_t gpsz, 22 unsigned long order) 23 { 24 gpa_t pos; 25 26 if (PTRS_PER_PTE < (gpsz >> order)) { 27 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 28 return; 29 } 30 31 if (has_svinval()) { 32 asm volatile (SFENCE_W_INVAL() ::: "memory"); 33 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) 34 asm volatile (HINVAL_GVMA(%0, %1) 35 : : "r" (pos >> 2), "r" (vmid) : "memory"); 36 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 37 } else { 38 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) 39 asm volatile (HFENCE_GVMA(%0, %1) 40 : : "r" (pos >> 2), "r" (vmid) : "memory"); 41 } 42 } 43 44 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) 45 { 46 asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory"); 47 } 48 49 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, 50 unsigned long order) 51 { 52 gpa_t pos; 53 54 if (PTRS_PER_PTE < (gpsz >> order)) { 55 kvm_riscv_local_hfence_gvma_all(); 56 return; 57 } 58 59 if (has_svinval()) { 60 asm volatile (SFENCE_W_INVAL() ::: "memory"); 61 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) 62 asm volatile(HINVAL_GVMA(%0, zero) 63 : : "r" (pos >> 2) : "memory"); 64 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 65 } else { 66 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) 67 asm volatile(HFENCE_GVMA(%0, zero) 68 : : "r" (pos >> 2) : "memory"); 69 } 70 } 71 72 void kvm_riscv_local_hfence_gvma_all(void) 73 { 74 asm volatile(HFENCE_GVMA(zero, zero) : : : "memory"); 75 } 76 77 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, 78 unsigned long asid, 79 unsigned long gva, 80 unsigned long gvsz, 81 unsigned long order) 82 { 83 unsigned long pos, hgatp; 84 85 if (PTRS_PER_PTE < (gvsz >> order)) { 86 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); 87 return; 88 } 89 90 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 91 92 if (has_svinval()) { 93 asm volatile (SFENCE_W_INVAL() ::: "memory"); 94 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) 95 asm volatile(HINVAL_VVMA(%0, %1) 96 : : "r" (pos), "r" (asid) : "memory"); 97 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 98 } else { 99 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) 100 asm volatile(HFENCE_VVMA(%0, %1) 101 : : "r" (pos), "r" (asid) : "memory"); 102 } 103 104 csr_write(CSR_HGATP, hgatp); 105 } 106 107 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, 108 unsigned long asid) 109 { 110 unsigned long hgatp; 111 112 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 113 114 asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory"); 115 116 csr_write(CSR_HGATP, hgatp); 117 } 118 119 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, 120 unsigned long gva, unsigned long gvsz, 121 unsigned long order) 122 { 123 unsigned long pos, hgatp; 124 125 if (PTRS_PER_PTE < (gvsz >> order)) { 126 kvm_riscv_local_hfence_vvma_all(vmid); 127 return; 128 } 129 130 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 131 132 if (has_svinval()) { 133 asm volatile (SFENCE_W_INVAL() ::: "memory"); 134 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) 135 asm volatile(HINVAL_VVMA(%0, zero) 136 : : "r" (pos) : "memory"); 137 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 138 } else { 139 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) 140 asm volatile(HFENCE_VVMA(%0, zero) 141 : : "r" (pos) : "memory"); 142 } 143 144 csr_write(CSR_HGATP, hgatp); 145 } 146 147 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) 148 { 149 unsigned long hgatp; 150 151 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 152 153 asm volatile(HFENCE_VVMA(zero, zero) : : : "memory"); 154 155 csr_write(CSR_HGATP, hgatp); 156 } 157 158 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) 159 { 160 unsigned long vmid; 161 162 if (!kvm_riscv_gstage_vmid_bits() || 163 vcpu->arch.last_exit_cpu == vcpu->cpu) 164 return; 165 166 /* 167 * On RISC-V platforms with hardware VMID support, we share same 168 * VMID for all VCPUs of a particular Guest/VM. This means we might 169 * have stale G-stage TLB entries on the current Host CPU due to 170 * some other VCPU of the same Guest which ran previously on the 171 * current Host CPU. 172 * 173 * To cleanup stale TLB entries, we simply flush all G-stage TLB 174 * entries by VMID whenever underlying Host CPU changes for a VCPU. 175 */ 176 177 vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); 178 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 179 } 180 181 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) 182 { 183 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD); 184 local_flush_icache_all(); 185 } 186 187 void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu) 188 { 189 struct kvm_vmid *vmid; 190 191 vmid = &vcpu->kvm->arch.vmid; 192 kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid)); 193 } 194 195 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) 196 { 197 struct kvm_vmid *vmid; 198 199 vmid = &vcpu->kvm->arch.vmid; 200 kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid)); 201 } 202 203 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu, 204 struct kvm_riscv_hfence *out_data) 205 { 206 bool ret = false; 207 struct kvm_vcpu_arch *varch = &vcpu->arch; 208 209 spin_lock(&varch->hfence_lock); 210 211 if (varch->hfence_queue[varch->hfence_head].type) { 212 memcpy(out_data, &varch->hfence_queue[varch->hfence_head], 213 sizeof(*out_data)); 214 varch->hfence_queue[varch->hfence_head].type = 0; 215 216 varch->hfence_head++; 217 if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE) 218 varch->hfence_head = 0; 219 220 ret = true; 221 } 222 223 spin_unlock(&varch->hfence_lock); 224 225 return ret; 226 } 227 228 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, 229 const struct kvm_riscv_hfence *data) 230 { 231 bool ret = false; 232 struct kvm_vcpu_arch *varch = &vcpu->arch; 233 234 spin_lock(&varch->hfence_lock); 235 236 if (!varch->hfence_queue[varch->hfence_tail].type) { 237 memcpy(&varch->hfence_queue[varch->hfence_tail], 238 data, sizeof(*data)); 239 240 varch->hfence_tail++; 241 if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE) 242 varch->hfence_tail = 0; 243 244 ret = true; 245 } 246 247 spin_unlock(&varch->hfence_lock); 248 249 return ret; 250 } 251 252 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) 253 { 254 struct kvm_riscv_hfence d = { 0 }; 255 struct kvm_vmid *v = &vcpu->kvm->arch.vmid; 256 257 while (vcpu_hfence_dequeue(vcpu, &d)) { 258 switch (d.type) { 259 case KVM_RISCV_HFENCE_UNKNOWN: 260 break; 261 case KVM_RISCV_HFENCE_GVMA_VMID_GPA: 262 kvm_riscv_local_hfence_gvma_vmid_gpa( 263 READ_ONCE(v->vmid), 264 d.addr, d.size, d.order); 265 break; 266 case KVM_RISCV_HFENCE_VVMA_ASID_GVA: 267 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); 268 kvm_riscv_local_hfence_vvma_asid_gva( 269 READ_ONCE(v->vmid), d.asid, 270 d.addr, d.size, d.order); 271 break; 272 case KVM_RISCV_HFENCE_VVMA_ASID_ALL: 273 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); 274 kvm_riscv_local_hfence_vvma_asid_all( 275 READ_ONCE(v->vmid), d.asid); 276 break; 277 case KVM_RISCV_HFENCE_VVMA_GVA: 278 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); 279 kvm_riscv_local_hfence_vvma_gva( 280 READ_ONCE(v->vmid), 281 d.addr, d.size, d.order); 282 break; 283 default: 284 break; 285 } 286 } 287 } 288 289 static void make_xfence_request(struct kvm *kvm, 290 unsigned long hbase, unsigned long hmask, 291 unsigned int req, unsigned int fallback_req, 292 const struct kvm_riscv_hfence *data) 293 { 294 unsigned long i; 295 struct kvm_vcpu *vcpu; 296 unsigned int actual_req = req; 297 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); 298 299 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 300 kvm_for_each_vcpu(i, vcpu, kvm) { 301 if (hbase != -1UL) { 302 if (vcpu->vcpu_id < hbase) 303 continue; 304 if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) 305 continue; 306 } 307 308 bitmap_set(vcpu_mask, i, 1); 309 310 if (!data || !data->type) 311 continue; 312 313 /* 314 * Enqueue hfence data to VCPU hfence queue. If we don't 315 * have space in the VCPU hfence queue then fallback to 316 * a more conservative hfence request. 317 */ 318 if (!vcpu_hfence_enqueue(vcpu, data)) 319 actual_req = fallback_req; 320 } 321 322 kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask); 323 } 324 325 void kvm_riscv_fence_i(struct kvm *kvm, 326 unsigned long hbase, unsigned long hmask) 327 { 328 make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I, 329 KVM_REQ_FENCE_I, NULL); 330 } 331 332 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, 333 unsigned long hbase, unsigned long hmask, 334 gpa_t gpa, gpa_t gpsz, 335 unsigned long order) 336 { 337 struct kvm_riscv_hfence data; 338 339 data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; 340 data.asid = 0; 341 data.addr = gpa; 342 data.size = gpsz; 343 data.order = order; 344 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 345 KVM_REQ_HFENCE_GVMA_VMID_ALL, &data); 346 } 347 348 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, 349 unsigned long hbase, unsigned long hmask) 350 { 351 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL, 352 KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL); 353 } 354 355 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, 356 unsigned long hbase, unsigned long hmask, 357 unsigned long gva, unsigned long gvsz, 358 unsigned long order, unsigned long asid) 359 { 360 struct kvm_riscv_hfence data; 361 362 data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; 363 data.asid = asid; 364 data.addr = gva; 365 data.size = gvsz; 366 data.order = order; 367 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 368 KVM_REQ_HFENCE_VVMA_ALL, &data); 369 } 370 371 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, 372 unsigned long hbase, unsigned long hmask, 373 unsigned long asid) 374 { 375 struct kvm_riscv_hfence data; 376 377 data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; 378 data.asid = asid; 379 data.addr = data.size = data.order = 0; 380 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 381 KVM_REQ_HFENCE_VVMA_ALL, &data); 382 } 383 384 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, 385 unsigned long hbase, unsigned long hmask, 386 unsigned long gva, unsigned long gvsz, 387 unsigned long order) 388 { 389 struct kvm_riscv_hfence data; 390 391 data.type = KVM_RISCV_HFENCE_VVMA_GVA; 392 data.asid = 0; 393 data.addr = gva; 394 data.size = gvsz; 395 data.order = order; 396 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 397 KVM_REQ_HFENCE_VVMA_ALL, &data); 398 } 399 400 void kvm_riscv_hfence_vvma_all(struct kvm *kvm, 401 unsigned long hbase, unsigned long hmask) 402 { 403 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL, 404 KVM_REQ_HFENCE_VVMA_ALL, NULL); 405 } 406