1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022 Ventana Micro Systems Inc. 4 */ 5 6 #include <linux/bitmap.h> 7 #include <linux/cpumask.h> 8 #include <linux/errno.h> 9 #include <linux/err.h> 10 #include <linux/module.h> 11 #include <linux/overflow.h> 12 #include <linux/smp.h> 13 #include <linux/kvm_host.h> 14 #include <asm/cacheflush.h> 15 #include <asm/csr.h> 16 #include <asm/cpufeature.h> 17 #include <asm/insn-def.h> 18 #include <asm/kvm_nacl.h> 19 #include <asm/kvm_tlb.h> 20 #include <asm/kvm_vmid.h> 21 22 #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) 23 24 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, 25 gpa_t gpa, gpa_t gpsz, 26 unsigned long order) 27 { 28 gpa_t end, pos, step = BIT(order); 29 30 if (check_add_overflow(gpa, gpsz, &end)) { 31 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 32 return; 33 } 34 35 if (PTRS_PER_PTE < (gpsz >> order)) { 36 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 37 return; 38 } 39 40 if (has_svinval()) { 41 asm volatile (SFENCE_W_INVAL() ::: "memory"); 42 for (pos = gpa; pos < end; pos += step) { 43 asm volatile (HINVAL_GVMA(%0, %1) 44 : : "r" (pos >> 2), "r" (vmid) : "memory"); 45 if (end - pos <= step) 46 break; 47 } 48 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 49 } else { 50 for (pos = gpa; pos < end; pos += step) { 51 asm volatile (HFENCE_GVMA(%0, %1) 52 : : "r" (pos >> 2), "r" (vmid) : "memory"); 53 if (end - pos <= step) 54 break; 55 } 56 } 57 } 58 59 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) 60 { 61 asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory"); 62 } 63 64 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, 65 unsigned long order) 66 { 67 gpa_t end, pos, step = BIT(order); 68 69 if (check_add_overflow(gpa, gpsz, &end)) { 70 kvm_riscv_local_hfence_gvma_all(); 71 return; 72 } 73 74 if (PTRS_PER_PTE < (gpsz >> order)) { 75 kvm_riscv_local_hfence_gvma_all(); 76 return; 77 } 78 79 if (has_svinval()) { 80 asm volatile (SFENCE_W_INVAL() ::: "memory"); 81 for (pos = gpa; pos < end; pos += step) { 82 asm volatile(HINVAL_GVMA(%0, zero) 83 : : "r" (pos >> 2) : "memory"); 84 if (end - pos <= step) 85 break; 86 } 87 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 88 } else { 89 for (pos = gpa; pos < end; pos += step) { 90 asm volatile(HFENCE_GVMA(%0, zero) 91 : : "r" (pos >> 2) : "memory"); 92 if (end - pos <= step) 93 break; 94 } 95 } 96 } 97 98 void kvm_riscv_local_hfence_gvma_all(void) 99 { 100 asm volatile(HFENCE_GVMA(zero, zero) : : : "memory"); 101 } 102 103 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, 104 unsigned long asid, 105 unsigned long gva, 106 unsigned long gvsz, 107 unsigned long order) 108 { 109 unsigned long end, pos, step = BIT(order); 110 unsigned long hgatp; 111 112 if (check_add_overflow(gva, gvsz, &end)) { 113 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); 114 return; 115 } 116 117 if (PTRS_PER_PTE < (gvsz >> order)) { 118 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); 119 return; 120 } 121 122 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 123 124 if (has_svinval()) { 125 asm volatile (SFENCE_W_INVAL() ::: "memory"); 126 for (pos = gva; pos < end; pos += step) { 127 asm volatile(HINVAL_VVMA(%0, %1) 128 : : "r" (pos), "r" (asid) : "memory"); 129 if (end - pos <= step) 130 break; 131 } 132 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 133 } else { 134 for (pos = gva; pos < end; pos += step) { 135 asm volatile(HFENCE_VVMA(%0, %1) 136 : : "r" (pos), "r" (asid) : "memory"); 137 if (end - pos <= step) 138 break; 139 } 140 } 141 142 csr_write(CSR_HGATP, hgatp); 143 } 144 145 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, 146 unsigned long asid) 147 { 148 unsigned long hgatp; 149 150 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 151 152 asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory"); 153 154 csr_write(CSR_HGATP, hgatp); 155 } 156 157 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, 158 unsigned long gva, unsigned long gvsz, 159 unsigned long order) 160 { 161 unsigned long end, pos, step = BIT(order); 162 unsigned long hgatp; 163 164 if (check_add_overflow(gva, gvsz, &end)) { 165 kvm_riscv_local_hfence_vvma_all(vmid); 166 return; 167 } 168 169 if (PTRS_PER_PTE < (gvsz >> order)) { 170 kvm_riscv_local_hfence_vvma_all(vmid); 171 return; 172 } 173 174 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 175 176 if (has_svinval()) { 177 asm volatile (SFENCE_W_INVAL() ::: "memory"); 178 for (pos = gva; pos < end; pos += step) { 179 asm volatile(HINVAL_VVMA(%0, zero) 180 : : "r" (pos) : "memory"); 181 if (end - pos <= step) 182 break; 183 } 184 asm volatile (SFENCE_INVAL_IR() ::: "memory"); 185 } else { 186 for (pos = gva; pos < end; pos += step) { 187 asm volatile(HFENCE_VVMA(%0, zero) 188 : : "r" (pos) : "memory"); 189 if (end - pos <= step) 190 break; 191 } 192 } 193 194 csr_write(CSR_HGATP, hgatp); 195 } 196 197 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) 198 { 199 unsigned long hgatp; 200 201 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 202 203 asm volatile(HFENCE_VVMA(zero, zero) : : : "memory"); 204 205 csr_write(CSR_HGATP, hgatp); 206 } 207 208 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) 209 { 210 unsigned long vmid; 211 212 if (!kvm_riscv_gstage_vmid_bits() || 213 vcpu->arch.last_exit_cpu == vcpu->cpu) 214 return; 215 216 /* 217 * On RISC-V platforms with hardware VMID support, we share same 218 * VMID for all VCPUs of a particular Guest/VM. This means we might 219 * have stale G-stage TLB entries on the current Host CPU due to 220 * some other VCPU of the same Guest which ran previously on the 221 * current Host CPU. 222 * 223 * To cleanup stale TLB entries, we simply flush all G-stage TLB 224 * entries by VMID whenever underlying Host CPU changes for a VCPU. 225 */ 226 227 vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); 228 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 229 230 /* 231 * Flush VS-stage TLB entries for implementation where VS-stage 232 * TLB does not cache guest physical address and VMID. 233 */ 234 if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa)) 235 kvm_riscv_local_hfence_vvma_all(vmid); 236 } 237 238 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) 239 { 240 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD); 241 local_flush_icache_all(); 242 } 243 244 void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu) 245 { 246 struct kvm_vmid *v = &vcpu->kvm->arch.vmid; 247 unsigned long vmid = READ_ONCE(v->vmid); 248 249 if (kvm_riscv_nacl_available()) 250 nacl_hfence_gvma_vmid_all(nacl_shmem(), vmid); 251 else 252 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 253 } 254 255 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) 256 { 257 struct kvm_vmid *v = &vcpu->kvm->arch.vmid; 258 unsigned long vmid = READ_ONCE(v->vmid); 259 260 if (kvm_riscv_nacl_available()) 261 nacl_hfence_vvma_all(nacl_shmem(), vmid); 262 else 263 kvm_riscv_local_hfence_vvma_all(vmid); 264 } 265 266 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu, 267 struct kvm_riscv_hfence *out_data) 268 { 269 bool ret = false; 270 struct kvm_vcpu_arch *varch = &vcpu->arch; 271 272 spin_lock(&varch->hfence_lock); 273 274 if (varch->hfence_queue[varch->hfence_head].type) { 275 memcpy(out_data, &varch->hfence_queue[varch->hfence_head], 276 sizeof(*out_data)); 277 varch->hfence_queue[varch->hfence_head].type = 0; 278 279 varch->hfence_head++; 280 if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE) 281 varch->hfence_head = 0; 282 283 ret = true; 284 } 285 286 spin_unlock(&varch->hfence_lock); 287 288 return ret; 289 } 290 291 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, 292 const struct kvm_riscv_hfence *data) 293 { 294 bool ret = false; 295 struct kvm_vcpu_arch *varch = &vcpu->arch; 296 297 spin_lock(&varch->hfence_lock); 298 299 if (!varch->hfence_queue[varch->hfence_tail].type) { 300 memcpy(&varch->hfence_queue[varch->hfence_tail], 301 data, sizeof(*data)); 302 303 varch->hfence_tail++; 304 if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE) 305 varch->hfence_tail = 0; 306 307 ret = true; 308 } 309 310 spin_unlock(&varch->hfence_lock); 311 312 return ret; 313 } 314 315 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) 316 { 317 struct kvm_riscv_hfence d = { 0 }; 318 319 while (vcpu_hfence_dequeue(vcpu, &d)) { 320 switch (d.type) { 321 case KVM_RISCV_HFENCE_UNKNOWN: 322 break; 323 case KVM_RISCV_HFENCE_GVMA_VMID_GPA: 324 if (kvm_riscv_nacl_available()) 325 nacl_hfence_gvma_vmid(nacl_shmem(), d.vmid, 326 d.addr, d.size, d.order); 327 else 328 kvm_riscv_local_hfence_gvma_vmid_gpa(d.vmid, d.addr, 329 d.size, d.order); 330 break; 331 case KVM_RISCV_HFENCE_GVMA_VMID_ALL: 332 if (kvm_riscv_nacl_available()) 333 nacl_hfence_gvma_vmid_all(nacl_shmem(), d.vmid); 334 else 335 kvm_riscv_local_hfence_gvma_vmid_all(d.vmid); 336 break; 337 case KVM_RISCV_HFENCE_VVMA_ASID_GVA: 338 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); 339 if (kvm_riscv_nacl_available()) 340 nacl_hfence_vvma_asid(nacl_shmem(), d.vmid, d.asid, 341 d.addr, d.size, d.order); 342 else 343 kvm_riscv_local_hfence_vvma_asid_gva(d.vmid, d.asid, d.addr, 344 d.size, d.order); 345 break; 346 case KVM_RISCV_HFENCE_VVMA_ASID_ALL: 347 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); 348 if (kvm_riscv_nacl_available()) 349 nacl_hfence_vvma_asid_all(nacl_shmem(), d.vmid, d.asid); 350 else 351 kvm_riscv_local_hfence_vvma_asid_all(d.vmid, d.asid); 352 break; 353 case KVM_RISCV_HFENCE_VVMA_GVA: 354 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); 355 if (kvm_riscv_nacl_available()) 356 nacl_hfence_vvma(nacl_shmem(), d.vmid, 357 d.addr, d.size, d.order); 358 else 359 kvm_riscv_local_hfence_vvma_gva(d.vmid, d.addr, 360 d.size, d.order); 361 break; 362 case KVM_RISCV_HFENCE_VVMA_ALL: 363 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); 364 if (kvm_riscv_nacl_available()) 365 nacl_hfence_vvma_all(nacl_shmem(), d.vmid); 366 else 367 kvm_riscv_local_hfence_vvma_all(d.vmid); 368 break; 369 default: 370 break; 371 } 372 } 373 } 374 375 static void make_xfence_request_nodata(struct kvm *kvm, unsigned long hbase, 376 unsigned long hmask, unsigned int req) 377 { 378 unsigned long i; 379 struct kvm_vcpu *vcpu; 380 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); 381 382 bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); 383 kvm_for_each_vcpu(i, vcpu, kvm) { 384 if (hbase != -1UL) { 385 if (vcpu->vcpu_id < hbase || 386 vcpu->vcpu_id >= hbase + BITS_PER_LONG) 387 continue; 388 if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) 389 continue; 390 } 391 392 bitmap_set(vcpu_mask, i, 1); 393 } 394 395 kvm_make_vcpus_request_mask(kvm, req, vcpu_mask); 396 } 397 398 static void make_xfence_request(struct kvm *kvm, 399 unsigned long hbase, unsigned long hmask, 400 unsigned int req, unsigned int fallback_req, 401 const struct kvm_riscv_hfence *data) 402 { 403 unsigned long i; 404 struct kvm_vcpu *vcpu; 405 DECLARE_BITMAP(req_vcpu_mask, KVM_MAX_VCPUS); 406 DECLARE_BITMAP(fallback_req_vcpu_mask, KVM_MAX_VCPUS); 407 408 if (!data || !data->type) 409 return; 410 411 bitmap_zero(req_vcpu_mask, KVM_MAX_VCPUS); 412 bitmap_zero(fallback_req_vcpu_mask, KVM_MAX_VCPUS); 413 kvm_for_each_vcpu(i, vcpu, kvm) { 414 if (hbase != -1UL) { 415 if (vcpu->vcpu_id < hbase || 416 vcpu->vcpu_id >= hbase + BITS_PER_LONG) 417 continue; 418 if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) 419 continue; 420 } 421 422 /* 423 * Enqueue hfence data to VCPU hfence queue. If we don't 424 * have space in the VCPU hfence queue then fallback to 425 * a more conservative hfence request. 426 */ 427 if (!vcpu_hfence_enqueue(vcpu, data)) 428 bitmap_set(fallback_req_vcpu_mask, i, 1); 429 else 430 bitmap_set(req_vcpu_mask, i, 1); 431 } 432 433 kvm_make_vcpus_request_mask(kvm, req, req_vcpu_mask); 434 kvm_make_vcpus_request_mask(kvm, fallback_req, fallback_req_vcpu_mask); 435 } 436 437 void kvm_riscv_fence_i(struct kvm *kvm, 438 unsigned long hbase, unsigned long hmask) 439 { 440 make_xfence_request_nodata(kvm, hbase, hmask, KVM_REQ_FENCE_I); 441 } 442 443 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, 444 unsigned long hbase, unsigned long hmask, 445 gpa_t gpa, gpa_t gpsz, 446 unsigned long order, unsigned long vmid) 447 { 448 struct kvm_riscv_hfence data; 449 450 data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; 451 data.asid = 0; 452 data.vmid = vmid; 453 data.addr = gpa; 454 data.size = gpsz; 455 data.order = order; 456 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 457 KVM_REQ_TLB_FLUSH, &data); 458 } 459 460 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, 461 unsigned long hbase, unsigned long hmask, 462 unsigned long vmid) 463 { 464 struct kvm_riscv_hfence data = {0}; 465 466 data.type = KVM_RISCV_HFENCE_GVMA_VMID_ALL; 467 data.vmid = vmid; 468 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 469 KVM_REQ_TLB_FLUSH, &data); 470 } 471 472 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, 473 unsigned long hbase, unsigned long hmask, 474 unsigned long gva, unsigned long gvsz, 475 unsigned long order, unsigned long asid, 476 unsigned long vmid) 477 { 478 struct kvm_riscv_hfence data; 479 480 data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; 481 data.asid = asid; 482 data.vmid = vmid; 483 data.addr = gva; 484 data.size = gvsz; 485 data.order = order; 486 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 487 KVM_REQ_HFENCE_VVMA_ALL, &data); 488 } 489 490 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, 491 unsigned long hbase, unsigned long hmask, 492 unsigned long asid, unsigned long vmid) 493 { 494 struct kvm_riscv_hfence data = {0}; 495 496 data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; 497 data.asid = asid; 498 data.vmid = vmid; 499 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 500 KVM_REQ_HFENCE_VVMA_ALL, &data); 501 } 502 503 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, 504 unsigned long hbase, unsigned long hmask, 505 unsigned long gva, unsigned long gvsz, 506 unsigned long order, unsigned long vmid) 507 { 508 struct kvm_riscv_hfence data; 509 510 data.type = KVM_RISCV_HFENCE_VVMA_GVA; 511 data.asid = 0; 512 data.vmid = vmid; 513 data.addr = gva; 514 data.size = gvsz; 515 data.order = order; 516 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 517 KVM_REQ_HFENCE_VVMA_ALL, &data); 518 } 519 520 void kvm_riscv_hfence_vvma_all(struct kvm *kvm, 521 unsigned long hbase, unsigned long hmask, 522 unsigned long vmid) 523 { 524 struct kvm_riscv_hfence data = {0}; 525 526 data.type = KVM_RISCV_HFENCE_VVMA_ALL; 527 data.vmid = vmid; 528 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 529 KVM_REQ_HFENCE_VVMA_ALL, &data); 530 } 531 532 int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages) 533 { 534 kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, 535 gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT, 536 PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid)); 537 return 0; 538 } 539