1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/errno.h> 10 #include <linux/hugetlb.h> 11 #include <linux/module.h> 12 #include <linux/uaccess.h> 13 #include <linux/vmalloc.h> 14 #include <linux/kvm_host.h> 15 #include <linux/sched/signal.h> 16 #include <asm/kvm_mmu.h> 17 #include <asm/kvm_nacl.h> 18 19 static void mmu_wp_memory_region(struct kvm *kvm, int slot) 20 { 21 struct kvm_memslots *slots = kvm_memslots(kvm); 22 struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); 23 phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; 24 phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; 25 struct kvm_gstage gstage; 26 27 gstage.kvm = kvm; 28 gstage.flags = 0; 29 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 30 gstage.pgd = kvm->arch.pgd; 31 32 spin_lock(&kvm->mmu_lock); 33 kvm_riscv_gstage_wp_range(&gstage, start, end); 34 spin_unlock(&kvm->mmu_lock); 35 kvm_flush_remote_tlbs_memslot(kvm, memslot); 36 } 37 38 int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, 39 unsigned long size, bool writable, bool in_atomic) 40 { 41 int ret = 0; 42 unsigned long pfn; 43 phys_addr_t addr, end; 44 struct kvm_mmu_memory_cache pcache = { 45 .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, 46 .gfp_zero = __GFP_ZERO, 47 }; 48 struct kvm_gstage_mapping map; 49 struct kvm_gstage gstage; 50 51 gstage.kvm = kvm; 52 gstage.flags = 0; 53 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 54 gstage.pgd = kvm->arch.pgd; 55 56 end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK; 57 pfn = __phys_to_pfn(hpa); 58 59 for (addr = gpa; addr < end; addr += PAGE_SIZE) { 60 map.addr = addr; 61 map.pte = pfn_pte(pfn, PAGE_KERNEL_IO); 62 map.level = 0; 63 64 if (!writable) 65 map.pte = pte_wrprotect(map.pte); 66 67 ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels); 68 if (ret) 69 goto out; 70 71 spin_lock(&kvm->mmu_lock); 72 ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map); 73 spin_unlock(&kvm->mmu_lock); 74 if (ret) 75 goto out; 76 77 pfn++; 78 } 79 80 out: 81 kvm_mmu_free_memory_cache(&pcache); 82 return ret; 83 } 84 85 void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) 86 { 87 struct kvm_gstage gstage; 88 89 gstage.kvm = kvm; 90 gstage.flags = 0; 91 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 92 gstage.pgd = kvm->arch.pgd; 93 94 spin_lock(&kvm->mmu_lock); 95 kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); 96 spin_unlock(&kvm->mmu_lock); 97 } 98 99 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 100 struct kvm_memory_slot *slot, 101 gfn_t gfn_offset, 102 unsigned long mask) 103 { 104 phys_addr_t base_gfn = slot->base_gfn + gfn_offset; 105 phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; 106 phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; 107 struct kvm_gstage gstage; 108 109 gstage.kvm = kvm; 110 gstage.flags = 0; 111 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 112 gstage.pgd = kvm->arch.pgd; 113 114 kvm_riscv_gstage_wp_range(&gstage, start, end); 115 } 116 117 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 118 { 119 } 120 121 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free) 122 { 123 } 124 125 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 126 { 127 } 128 129 void kvm_arch_flush_shadow_all(struct kvm *kvm) 130 { 131 kvm_riscv_mmu_free_pgd(kvm); 132 } 133 134 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 135 struct kvm_memory_slot *slot) 136 { 137 gpa_t gpa = slot->base_gfn << PAGE_SHIFT; 138 phys_addr_t size = slot->npages << PAGE_SHIFT; 139 struct kvm_gstage gstage; 140 141 gstage.kvm = kvm; 142 gstage.flags = 0; 143 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 144 gstage.pgd = kvm->arch.pgd; 145 146 spin_lock(&kvm->mmu_lock); 147 kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); 148 spin_unlock(&kvm->mmu_lock); 149 } 150 151 void kvm_arch_commit_memory_region(struct kvm *kvm, 152 struct kvm_memory_slot *old, 153 const struct kvm_memory_slot *new, 154 enum kvm_mr_change change) 155 { 156 /* 157 * At this point memslot has been committed and there is an 158 * allocated dirty_bitmap[], dirty pages will be tracked while 159 * the memory slot is write protected. 160 */ 161 if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) 162 mmu_wp_memory_region(kvm, new->id); 163 } 164 165 int kvm_arch_prepare_memory_region(struct kvm *kvm, 166 const struct kvm_memory_slot *old, 167 struct kvm_memory_slot *new, 168 enum kvm_mr_change change) 169 { 170 hva_t hva, reg_end, size; 171 gpa_t base_gpa; 172 bool writable; 173 int ret = 0; 174 175 if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && 176 change != KVM_MR_FLAGS_ONLY) 177 return 0; 178 179 /* 180 * Prevent userspace from creating a memory region outside of the GPA 181 * space addressable by the KVM guest GPA space. 182 */ 183 if ((new->base_gfn + new->npages) >= 184 (kvm_riscv_gstage_gpa_size >> PAGE_SHIFT)) 185 return -EFAULT; 186 187 hva = new->userspace_addr; 188 size = new->npages << PAGE_SHIFT; 189 reg_end = hva + size; 190 base_gpa = new->base_gfn << PAGE_SHIFT; 191 writable = !(new->flags & KVM_MEM_READONLY); 192 193 mmap_read_lock(current->mm); 194 195 /* 196 * A memory region could potentially cover multiple VMAs, and 197 * any holes between them, so iterate over all of them to find 198 * out if we can map any of them right now. 199 * 200 * +--------------------------------------------+ 201 * +---------------+----------------+ +----------------+ 202 * | : VMA 1 | VMA 2 | | VMA 3 : | 203 * +---------------+----------------+ +----------------+ 204 * | memory region | 205 * +--------------------------------------------+ 206 */ 207 do { 208 struct vm_area_struct *vma; 209 hva_t vm_start, vm_end; 210 211 vma = find_vma_intersection(current->mm, hva, reg_end); 212 if (!vma) 213 break; 214 215 /* 216 * Mapping a read-only VMA is only allowed if the 217 * memory region is configured as read-only. 218 */ 219 if (writable && !(vma->vm_flags & VM_WRITE)) { 220 ret = -EPERM; 221 break; 222 } 223 224 /* Take the intersection of this VMA with the memory region */ 225 vm_start = max(hva, vma->vm_start); 226 vm_end = min(reg_end, vma->vm_end); 227 228 if (vma->vm_flags & VM_PFNMAP) { 229 gpa_t gpa = base_gpa + (vm_start - hva); 230 phys_addr_t pa; 231 232 pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; 233 pa += vm_start - vma->vm_start; 234 235 /* IO region dirty page logging not allowed */ 236 if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { 237 ret = -EINVAL; 238 goto out; 239 } 240 241 ret = kvm_riscv_mmu_ioremap(kvm, gpa, pa, vm_end - vm_start, 242 writable, false); 243 if (ret) 244 break; 245 } 246 hva = vm_end; 247 } while (hva < reg_end); 248 249 if (change == KVM_MR_FLAGS_ONLY) 250 goto out; 251 252 if (ret) 253 kvm_riscv_mmu_iounmap(kvm, base_gpa, size); 254 255 out: 256 mmap_read_unlock(current->mm); 257 return ret; 258 } 259 260 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 261 { 262 struct kvm_gstage gstage; 263 264 if (!kvm->arch.pgd) 265 return false; 266 267 gstage.kvm = kvm; 268 gstage.flags = 0; 269 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 270 gstage.pgd = kvm->arch.pgd; 271 kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT, 272 (range->end - range->start) << PAGE_SHIFT, 273 range->may_block); 274 return false; 275 } 276 277 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 278 { 279 pte_t *ptep; 280 u32 ptep_level = 0; 281 u64 size = (range->end - range->start) << PAGE_SHIFT; 282 struct kvm_gstage gstage; 283 284 if (!kvm->arch.pgd) 285 return false; 286 287 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 288 289 gstage.kvm = kvm; 290 gstage.flags = 0; 291 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 292 gstage.pgd = kvm->arch.pgd; 293 if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, 294 &ptep, &ptep_level)) 295 return false; 296 297 return ptep_test_and_clear_young(NULL, 0, ptep); 298 } 299 300 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 301 { 302 pte_t *ptep; 303 u32 ptep_level = 0; 304 u64 size = (range->end - range->start) << PAGE_SHIFT; 305 struct kvm_gstage gstage; 306 307 if (!kvm->arch.pgd) 308 return false; 309 310 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 311 312 gstage.kvm = kvm; 313 gstage.flags = 0; 314 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 315 gstage.pgd = kvm->arch.pgd; 316 if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, 317 &ptep, &ptep_level)) 318 return false; 319 320 return pte_young(ptep_get(ptep)); 321 } 322 323 int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, 324 gpa_t gpa, unsigned long hva, bool is_write, 325 struct kvm_gstage_mapping *out_map) 326 { 327 int ret; 328 kvm_pfn_t hfn; 329 bool writable; 330 short vma_pageshift; 331 gfn_t gfn = gpa >> PAGE_SHIFT; 332 struct vm_area_struct *vma; 333 struct kvm *kvm = vcpu->kvm; 334 struct kvm_mmu_memory_cache *pcache = &vcpu->arch.mmu_page_cache; 335 bool logging = (memslot->dirty_bitmap && 336 !(memslot->flags & KVM_MEM_READONLY)) ? true : false; 337 unsigned long vma_pagesize, mmu_seq; 338 struct kvm_gstage gstage; 339 struct page *page; 340 341 gstage.kvm = kvm; 342 gstage.flags = 0; 343 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 344 gstage.pgd = kvm->arch.pgd; 345 346 /* Setup initial state of output mapping */ 347 memset(out_map, 0, sizeof(*out_map)); 348 349 /* We need minimum second+third level pages */ 350 ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels); 351 if (ret) { 352 kvm_err("Failed to topup G-stage cache\n"); 353 return ret; 354 } 355 356 mmap_read_lock(current->mm); 357 358 vma = vma_lookup(current->mm, hva); 359 if (unlikely(!vma)) { 360 kvm_err("Failed to find VMA for hva 0x%lx\n", hva); 361 mmap_read_unlock(current->mm); 362 return -EFAULT; 363 } 364 365 if (is_vm_hugetlb_page(vma)) 366 vma_pageshift = huge_page_shift(hstate_vma(vma)); 367 else 368 vma_pageshift = PAGE_SHIFT; 369 vma_pagesize = 1ULL << vma_pageshift; 370 if (logging || (vma->vm_flags & VM_PFNMAP)) 371 vma_pagesize = PAGE_SIZE; 372 373 if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) 374 gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; 375 376 /* 377 * Read mmu_invalidate_seq so that KVM can detect if the results of 378 * vma_lookup() or __kvm_faultin_pfn() become stale prior to acquiring 379 * kvm->mmu_lock. 380 * 381 * Rely on mmap_read_unlock() for an implicit smp_rmb(), which pairs 382 * with the smp_wmb() in kvm_mmu_invalidate_end(). 383 */ 384 mmu_seq = kvm->mmu_invalidate_seq; 385 mmap_read_unlock(current->mm); 386 387 if (vma_pagesize != PUD_SIZE && 388 vma_pagesize != PMD_SIZE && 389 vma_pagesize != PAGE_SIZE) { 390 kvm_err("Invalid VMA page size 0x%lx\n", vma_pagesize); 391 return -EFAULT; 392 } 393 394 hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0, 395 &writable, &page); 396 if (hfn == KVM_PFN_ERR_HWPOISON) { 397 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva, 398 vma_pageshift, current); 399 return 0; 400 } 401 if (is_error_noslot_pfn(hfn)) 402 return -EFAULT; 403 404 /* 405 * If logging is active then we allow writable pages only 406 * for write faults. 407 */ 408 if (logging && !is_write) 409 writable = false; 410 411 spin_lock(&kvm->mmu_lock); 412 413 if (mmu_invalidate_retry(kvm, mmu_seq)) 414 goto out_unlock; 415 416 if (writable) { 417 mark_page_dirty_in_slot(kvm, memslot, gfn); 418 ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, 419 vma_pagesize, false, true, out_map); 420 } else { 421 ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, 422 vma_pagesize, true, true, out_map); 423 } 424 425 if (ret) 426 kvm_err("Failed to map in G-stage\n"); 427 428 out_unlock: 429 kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); 430 spin_unlock(&kvm->mmu_lock); 431 return ret; 432 } 433 434 int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) 435 { 436 struct page *pgd_page; 437 438 if (kvm->arch.pgd != NULL) { 439 kvm_err("kvm_arch already initialized?\n"); 440 return -EINVAL; 441 } 442 443 pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 444 get_order(kvm_riscv_gstage_pgd_size)); 445 if (!pgd_page) 446 return -ENOMEM; 447 kvm->arch.pgd = page_to_virt(pgd_page); 448 kvm->arch.pgd_phys = page_to_phys(pgd_page); 449 450 return 0; 451 } 452 453 void kvm_riscv_mmu_free_pgd(struct kvm *kvm) 454 { 455 struct kvm_gstage gstage; 456 void *pgd = NULL; 457 458 spin_lock(&kvm->mmu_lock); 459 if (kvm->arch.pgd) { 460 gstage.kvm = kvm; 461 gstage.flags = 0; 462 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 463 gstage.pgd = kvm->arch.pgd; 464 kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false); 465 pgd = READ_ONCE(kvm->arch.pgd); 466 kvm->arch.pgd = NULL; 467 kvm->arch.pgd_phys = 0; 468 } 469 spin_unlock(&kvm->mmu_lock); 470 471 if (pgd) 472 free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size)); 473 } 474 475 void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) 476 { 477 unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT; 478 struct kvm_arch *k = &vcpu->kvm->arch; 479 480 hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; 481 hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; 482 483 ncsr_write(CSR_HGATP, hgatp); 484 485 if (!kvm_riscv_gstage_vmid_bits()) 486 kvm_riscv_local_hfence_gvma_all(); 487 } 488