1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/errno.h> 10 #include <linux/hugetlb.h> 11 #include <linux/module.h> 12 #include <linux/uaccess.h> 13 #include <linux/vmalloc.h> 14 #include <linux/kvm_host.h> 15 #include <linux/sched/signal.h> 16 #include <asm/kvm_mmu.h> 17 #include <asm/kvm_nacl.h> 18 19 static void mmu_wp_memory_region(struct kvm *kvm, int slot) 20 { 21 struct kvm_memslots *slots = kvm_memslots(kvm); 22 struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); 23 phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; 24 phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; 25 struct kvm_gstage gstage; 26 27 gstage.kvm = kvm; 28 gstage.flags = 0; 29 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 30 gstage.pgd = kvm->arch.pgd; 31 32 spin_lock(&kvm->mmu_lock); 33 kvm_riscv_gstage_wp_range(&gstage, start, end); 34 spin_unlock(&kvm->mmu_lock); 35 kvm_flush_remote_tlbs_memslot(kvm, memslot); 36 } 37 38 int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, 39 unsigned long size, bool writable, bool in_atomic) 40 { 41 int ret = 0; 42 pgprot_t prot; 43 unsigned long pfn; 44 phys_addr_t addr, end; 45 struct kvm_mmu_memory_cache pcache = { 46 .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, 47 .gfp_zero = __GFP_ZERO, 48 }; 49 struct kvm_gstage_mapping map; 50 struct kvm_gstage gstage; 51 52 gstage.kvm = kvm; 53 gstage.flags = 0; 54 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 55 gstage.pgd = kvm->arch.pgd; 56 57 end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK; 58 pfn = __phys_to_pfn(hpa); 59 prot = pgprot_noncached(PAGE_WRITE); 60 61 for (addr = gpa; addr < end; addr += PAGE_SIZE) { 62 map.addr = addr; 63 map.pte = pfn_pte(pfn, prot); 64 map.pte = pte_mkdirty(map.pte); 65 map.level = 0; 66 67 if (!writable) 68 map.pte = pte_wrprotect(map.pte); 69 70 ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels); 71 if (ret) 72 goto out; 73 74 spin_lock(&kvm->mmu_lock); 75 ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map); 76 spin_unlock(&kvm->mmu_lock); 77 if (ret) 78 goto out; 79 80 pfn++; 81 } 82 83 out: 84 kvm_mmu_free_memory_cache(&pcache); 85 return ret; 86 } 87 88 void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) 89 { 90 struct kvm_gstage gstage; 91 92 gstage.kvm = kvm; 93 gstage.flags = 0; 94 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 95 gstage.pgd = kvm->arch.pgd; 96 97 spin_lock(&kvm->mmu_lock); 98 kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); 99 spin_unlock(&kvm->mmu_lock); 100 } 101 102 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 103 struct kvm_memory_slot *slot, 104 gfn_t gfn_offset, 105 unsigned long mask) 106 { 107 phys_addr_t base_gfn = slot->base_gfn + gfn_offset; 108 phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; 109 phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; 110 struct kvm_gstage gstage; 111 112 gstage.kvm = kvm; 113 gstage.flags = 0; 114 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 115 gstage.pgd = kvm->arch.pgd; 116 117 kvm_riscv_gstage_wp_range(&gstage, start, end); 118 } 119 120 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 121 { 122 } 123 124 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free) 125 { 126 } 127 128 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 129 { 130 } 131 132 void kvm_arch_flush_shadow_all(struct kvm *kvm) 133 { 134 kvm_riscv_mmu_free_pgd(kvm); 135 } 136 137 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 138 struct kvm_memory_slot *slot) 139 { 140 gpa_t gpa = slot->base_gfn << PAGE_SHIFT; 141 phys_addr_t size = slot->npages << PAGE_SHIFT; 142 struct kvm_gstage gstage; 143 144 gstage.kvm = kvm; 145 gstage.flags = 0; 146 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 147 gstage.pgd = kvm->arch.pgd; 148 149 spin_lock(&kvm->mmu_lock); 150 kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); 151 spin_unlock(&kvm->mmu_lock); 152 } 153 154 void kvm_arch_commit_memory_region(struct kvm *kvm, 155 struct kvm_memory_slot *old, 156 const struct kvm_memory_slot *new, 157 enum kvm_mr_change change) 158 { 159 /* 160 * At this point memslot has been committed and there is an 161 * allocated dirty_bitmap[], dirty pages will be tracked while 162 * the memory slot is write protected. 163 */ 164 if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) 165 mmu_wp_memory_region(kvm, new->id); 166 } 167 168 int kvm_arch_prepare_memory_region(struct kvm *kvm, 169 const struct kvm_memory_slot *old, 170 struct kvm_memory_slot *new, 171 enum kvm_mr_change change) 172 { 173 hva_t hva, reg_end, size; 174 bool writable; 175 int ret = 0; 176 177 if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && 178 change != KVM_MR_FLAGS_ONLY) 179 return 0; 180 181 /* 182 * Prevent userspace from creating a memory region outside of the GPA 183 * space addressable by the KVM guest GPA space. 184 */ 185 if ((new->base_gfn + new->npages) >= 186 (kvm_riscv_gstage_gpa_size >> PAGE_SHIFT)) 187 return -EFAULT; 188 189 hva = new->userspace_addr; 190 size = new->npages << PAGE_SHIFT; 191 reg_end = hva + size; 192 writable = !(new->flags & KVM_MEM_READONLY); 193 194 mmap_read_lock(current->mm); 195 196 /* 197 * A memory region could potentially cover multiple VMAs, and 198 * any holes between them, so iterate over all of them. 199 * 200 * +--------------------------------------------+ 201 * +---------------+----------------+ +----------------+ 202 * | : VMA 1 | VMA 2 | | VMA 3 : | 203 * +---------------+----------------+ +----------------+ 204 * | memory region | 205 * +--------------------------------------------+ 206 */ 207 do { 208 struct vm_area_struct *vma; 209 hva_t vm_end; 210 211 vma = find_vma_intersection(current->mm, hva, reg_end); 212 if (!vma) 213 break; 214 215 /* 216 * Mapping a read-only VMA is only allowed if the 217 * memory region is configured as read-only. 218 */ 219 if (writable && !(vma->vm_flags & VM_WRITE)) { 220 ret = -EPERM; 221 break; 222 } 223 224 /* Take the intersection of this VMA with the memory region */ 225 vm_end = min(reg_end, vma->vm_end); 226 227 if (vma->vm_flags & VM_PFNMAP) { 228 /* IO region dirty page logging not allowed */ 229 if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { 230 ret = -EINVAL; 231 goto out; 232 } 233 } 234 hva = vm_end; 235 } while (hva < reg_end); 236 237 out: 238 mmap_read_unlock(current->mm); 239 return ret; 240 } 241 242 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 243 { 244 struct kvm_gstage gstage; 245 246 if (!kvm->arch.pgd) 247 return false; 248 249 gstage.kvm = kvm; 250 gstage.flags = 0; 251 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 252 gstage.pgd = kvm->arch.pgd; 253 kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT, 254 (range->end - range->start) << PAGE_SHIFT, 255 range->may_block); 256 return false; 257 } 258 259 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 260 { 261 pte_t *ptep; 262 u32 ptep_level = 0; 263 u64 size = (range->end - range->start) << PAGE_SHIFT; 264 struct kvm_gstage gstage; 265 266 if (!kvm->arch.pgd) 267 return false; 268 269 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 270 271 gstage.kvm = kvm; 272 gstage.flags = 0; 273 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 274 gstage.pgd = kvm->arch.pgd; 275 if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, 276 &ptep, &ptep_level)) 277 return false; 278 279 return ptep_test_and_clear_young(NULL, 0, ptep); 280 } 281 282 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 283 { 284 pte_t *ptep; 285 u32 ptep_level = 0; 286 u64 size = (range->end - range->start) << PAGE_SHIFT; 287 struct kvm_gstage gstage; 288 289 if (!kvm->arch.pgd) 290 return false; 291 292 WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); 293 294 gstage.kvm = kvm; 295 gstage.flags = 0; 296 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 297 gstage.pgd = kvm->arch.pgd; 298 if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, 299 &ptep, &ptep_level)) 300 return false; 301 302 return pte_young(ptep_get(ptep)); 303 } 304 305 int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, 306 gpa_t gpa, unsigned long hva, bool is_write, 307 struct kvm_gstage_mapping *out_map) 308 { 309 int ret; 310 kvm_pfn_t hfn; 311 bool writable; 312 short vma_pageshift; 313 gfn_t gfn = gpa >> PAGE_SHIFT; 314 struct vm_area_struct *vma; 315 struct kvm *kvm = vcpu->kvm; 316 struct kvm_mmu_memory_cache *pcache = &vcpu->arch.mmu_page_cache; 317 bool logging = (memslot->dirty_bitmap && 318 !(memslot->flags & KVM_MEM_READONLY)) ? true : false; 319 unsigned long vma_pagesize, mmu_seq; 320 struct kvm_gstage gstage; 321 struct page *page; 322 323 gstage.kvm = kvm; 324 gstage.flags = 0; 325 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 326 gstage.pgd = kvm->arch.pgd; 327 328 /* Setup initial state of output mapping */ 329 memset(out_map, 0, sizeof(*out_map)); 330 331 /* We need minimum second+third level pages */ 332 ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels); 333 if (ret) { 334 kvm_err("Failed to topup G-stage cache\n"); 335 return ret; 336 } 337 338 mmap_read_lock(current->mm); 339 340 vma = vma_lookup(current->mm, hva); 341 if (unlikely(!vma)) { 342 kvm_err("Failed to find VMA for hva 0x%lx\n", hva); 343 mmap_read_unlock(current->mm); 344 return -EFAULT; 345 } 346 347 if (is_vm_hugetlb_page(vma)) 348 vma_pageshift = huge_page_shift(hstate_vma(vma)); 349 else 350 vma_pageshift = PAGE_SHIFT; 351 vma_pagesize = 1ULL << vma_pageshift; 352 if (logging || (vma->vm_flags & VM_PFNMAP)) 353 vma_pagesize = PAGE_SIZE; 354 355 if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) 356 gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; 357 358 /* 359 * Read mmu_invalidate_seq so that KVM can detect if the results of 360 * vma_lookup() or __kvm_faultin_pfn() become stale prior to acquiring 361 * kvm->mmu_lock. 362 * 363 * Rely on mmap_read_unlock() for an implicit smp_rmb(), which pairs 364 * with the smp_wmb() in kvm_mmu_invalidate_end(). 365 */ 366 mmu_seq = kvm->mmu_invalidate_seq; 367 mmap_read_unlock(current->mm); 368 369 if (vma_pagesize != PUD_SIZE && 370 vma_pagesize != PMD_SIZE && 371 vma_pagesize != PAGE_SIZE) { 372 kvm_err("Invalid VMA page size 0x%lx\n", vma_pagesize); 373 return -EFAULT; 374 } 375 376 hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0, 377 &writable, &page); 378 if (hfn == KVM_PFN_ERR_HWPOISON) { 379 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva, 380 vma_pageshift, current); 381 return 0; 382 } 383 if (is_error_noslot_pfn(hfn)) 384 return -EFAULT; 385 386 /* 387 * If logging is active then we allow writable pages only 388 * for write faults. 389 */ 390 if (logging && !is_write) 391 writable = false; 392 393 spin_lock(&kvm->mmu_lock); 394 395 if (mmu_invalidate_retry(kvm, mmu_seq)) 396 goto out_unlock; 397 398 if (writable) { 399 mark_page_dirty_in_slot(kvm, memslot, gfn); 400 ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, 401 vma_pagesize, false, true, out_map); 402 } else { 403 ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, 404 vma_pagesize, true, true, out_map); 405 } 406 407 if (ret) 408 kvm_err("Failed to map in G-stage\n"); 409 410 out_unlock: 411 kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); 412 spin_unlock(&kvm->mmu_lock); 413 return ret; 414 } 415 416 int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) 417 { 418 struct page *pgd_page; 419 420 if (kvm->arch.pgd != NULL) { 421 kvm_err("kvm_arch already initialized?\n"); 422 return -EINVAL; 423 } 424 425 pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 426 get_order(kvm_riscv_gstage_pgd_size)); 427 if (!pgd_page) 428 return -ENOMEM; 429 kvm->arch.pgd = page_to_virt(pgd_page); 430 kvm->arch.pgd_phys = page_to_phys(pgd_page); 431 432 return 0; 433 } 434 435 void kvm_riscv_mmu_free_pgd(struct kvm *kvm) 436 { 437 struct kvm_gstage gstage; 438 void *pgd = NULL; 439 440 spin_lock(&kvm->mmu_lock); 441 if (kvm->arch.pgd) { 442 gstage.kvm = kvm; 443 gstage.flags = 0; 444 gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); 445 gstage.pgd = kvm->arch.pgd; 446 kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false); 447 pgd = READ_ONCE(kvm->arch.pgd); 448 kvm->arch.pgd = NULL; 449 kvm->arch.pgd_phys = 0; 450 } 451 spin_unlock(&kvm->mmu_lock); 452 453 if (pgd) 454 free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size)); 455 } 456 457 void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) 458 { 459 unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT; 460 struct kvm_arch *k = &vcpu->kvm->arch; 461 462 hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; 463 hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; 464 465 ncsr_write(CSR_HGATP, hgatp); 466 467 if (!kvm_riscv_gstage_vmid_bits()) 468 kvm_riscv_local_hfence_gvma_all(); 469 } 470