1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables kernel and guest-mode vCPU access to guest physical 6 * memory with suitable invalidation mechanisms. 7 * 8 * Copyright © 2021 Amazon.com, Inc. or its affiliates. 9 * 10 * Authors: 11 * David Woodhouse <dwmw2@infradead.org> 12 */ 13 14 #include <linux/kvm_host.h> 15 #include <linux/kvm.h> 16 #include <linux/highmem.h> 17 #include <linux/module.h> 18 #include <linux/errno.h> 19 20 #include "kvm_mm.h" 21 22 /* 23 * MMU notifier 'invalidate_range_start' hook. 24 */ 25 void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start, 26 unsigned long end, bool may_block) 27 { 28 DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS); 29 struct gfn_to_pfn_cache *gpc; 30 bool evict_vcpus = false; 31 32 spin_lock(&kvm->gpc_lock); 33 list_for_each_entry(gpc, &kvm->gpc_list, list) { 34 write_lock_irq(&gpc->lock); 35 36 /* Only a single page so no need to care about length */ 37 if (gpc->valid && !is_error_noslot_pfn(gpc->pfn) && 38 gpc->uhva >= start && gpc->uhva < end) { 39 gpc->valid = false; 40 41 /* 42 * If a guest vCPU could be using the physical address, 43 * it needs to be forced out of guest mode. 44 */ 45 if (gpc->usage & KVM_GUEST_USES_PFN) { 46 if (!evict_vcpus) { 47 evict_vcpus = true; 48 bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS); 49 } 50 __set_bit(gpc->vcpu->vcpu_idx, vcpu_bitmap); 51 } 52 } 53 write_unlock_irq(&gpc->lock); 54 } 55 spin_unlock(&kvm->gpc_lock); 56 57 if (evict_vcpus) { 58 /* 59 * KVM needs to ensure the vCPU is fully out of guest context 60 * before allowing the invalidation to continue. 61 */ 62 unsigned int req = KVM_REQ_OUTSIDE_GUEST_MODE; 63 bool called; 64 65 /* 66 * If the OOM reaper is active, then all vCPUs should have 67 * been stopped already, so perform the request without 68 * KVM_REQUEST_WAIT and be sad if any needed to be IPI'd. 69 */ 70 if (!may_block) 71 req &= ~KVM_REQUEST_WAIT; 72 73 called = kvm_make_vcpus_request_mask(kvm, req, vcpu_bitmap); 74 75 WARN_ON_ONCE(called && !may_block); 76 } 77 } 78 79 bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 80 gpa_t gpa, unsigned long len) 81 { 82 struct kvm_memslots *slots = kvm_memslots(kvm); 83 84 if ((gpa & ~PAGE_MASK) + len > PAGE_SIZE) 85 return false; 86 87 if (gpc->gpa != gpa || gpc->generation != slots->generation || 88 kvm_is_error_hva(gpc->uhva)) 89 return false; 90 91 if (!gpc->valid) 92 return false; 93 94 return true; 95 } 96 EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_check); 97 98 static void __release_gpc(struct kvm *kvm, kvm_pfn_t pfn, void *khva, gpa_t gpa) 99 { 100 /* Unmap the old page if it was mapped before, and release it */ 101 if (!is_error_noslot_pfn(pfn)) { 102 if (khva) { 103 if (pfn_valid(pfn)) 104 kunmap(pfn_to_page(pfn)); 105 #ifdef CONFIG_HAS_IOMEM 106 else 107 memunmap(khva); 108 #endif 109 } 110 111 kvm_release_pfn(pfn, false); 112 } 113 } 114 115 static kvm_pfn_t hva_to_pfn_retry(struct kvm *kvm, unsigned long uhva) 116 { 117 unsigned long mmu_seq; 118 kvm_pfn_t new_pfn; 119 int retry; 120 121 do { 122 mmu_seq = kvm->mmu_notifier_seq; 123 smp_rmb(); 124 125 /* We always request a writeable mapping */ 126 new_pfn = hva_to_pfn(uhva, false, NULL, true, NULL); 127 if (is_error_noslot_pfn(new_pfn)) 128 break; 129 130 KVM_MMU_READ_LOCK(kvm); 131 retry = mmu_notifier_retry_hva(kvm, mmu_seq, uhva); 132 KVM_MMU_READ_UNLOCK(kvm); 133 if (!retry) 134 break; 135 136 cond_resched(); 137 } while (1); 138 139 return new_pfn; 140 } 141 142 int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 143 gpa_t gpa, unsigned long len) 144 { 145 struct kvm_memslots *slots = kvm_memslots(kvm); 146 unsigned long page_offset = gpa & ~PAGE_MASK; 147 kvm_pfn_t old_pfn, new_pfn; 148 unsigned long old_uhva; 149 gpa_t old_gpa; 150 void *old_khva; 151 bool old_valid; 152 int ret = 0; 153 154 /* 155 * If must fit within a single page. The 'len' argument is 156 * only to enforce that. 157 */ 158 if (page_offset + len > PAGE_SIZE) 159 return -EINVAL; 160 161 write_lock_irq(&gpc->lock); 162 163 old_gpa = gpc->gpa; 164 old_pfn = gpc->pfn; 165 old_khva = gpc->khva - offset_in_page(gpc->khva); 166 old_uhva = gpc->uhva; 167 old_valid = gpc->valid; 168 169 /* If the userspace HVA is invalid, refresh that first */ 170 if (gpc->gpa != gpa || gpc->generation != slots->generation || 171 kvm_is_error_hva(gpc->uhva)) { 172 gfn_t gfn = gpa_to_gfn(gpa); 173 174 gpc->gpa = gpa; 175 gpc->generation = slots->generation; 176 gpc->memslot = __gfn_to_memslot(slots, gfn); 177 gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn); 178 179 if (kvm_is_error_hva(gpc->uhva)) { 180 gpc->pfn = KVM_PFN_ERR_FAULT; 181 ret = -EFAULT; 182 goto out; 183 } 184 185 gpc->uhva += page_offset; 186 } 187 188 /* 189 * If the userspace HVA changed or the PFN was already invalid, 190 * drop the lock and do the HVA to PFN lookup again. 191 */ 192 if (!old_valid || old_uhva != gpc->uhva) { 193 unsigned long uhva = gpc->uhva; 194 void *new_khva = NULL; 195 196 /* Placeholders for "hva is valid but not yet mapped" */ 197 gpc->pfn = KVM_PFN_ERR_FAULT; 198 gpc->khva = NULL; 199 gpc->valid = true; 200 201 write_unlock_irq(&gpc->lock); 202 203 new_pfn = hva_to_pfn_retry(kvm, uhva); 204 if (is_error_noslot_pfn(new_pfn)) { 205 ret = -EFAULT; 206 goto map_done; 207 } 208 209 if (gpc->usage & KVM_HOST_USES_PFN) { 210 if (new_pfn == old_pfn) { 211 new_khva = old_khva; 212 old_pfn = KVM_PFN_ERR_FAULT; 213 old_khva = NULL; 214 } else if (pfn_valid(new_pfn)) { 215 new_khva = kmap(pfn_to_page(new_pfn)); 216 #ifdef CONFIG_HAS_IOMEM 217 } else { 218 new_khva = memremap(pfn_to_hpa(new_pfn), PAGE_SIZE, MEMREMAP_WB); 219 #endif 220 } 221 if (new_khva) 222 new_khva += page_offset; 223 else 224 ret = -EFAULT; 225 } 226 227 map_done: 228 write_lock_irq(&gpc->lock); 229 if (ret) { 230 gpc->valid = false; 231 gpc->pfn = KVM_PFN_ERR_FAULT; 232 gpc->khva = NULL; 233 } else { 234 /* At this point, gpc->valid may already have been cleared */ 235 gpc->pfn = new_pfn; 236 gpc->khva = new_khva; 237 } 238 } else { 239 /* If the HVA→PFN mapping was already valid, don't unmap it. */ 240 old_pfn = KVM_PFN_ERR_FAULT; 241 old_khva = NULL; 242 } 243 244 out: 245 write_unlock_irq(&gpc->lock); 246 247 __release_gpc(kvm, old_pfn, old_khva, old_gpa); 248 249 return ret; 250 } 251 EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_refresh); 252 253 void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) 254 { 255 void *old_khva; 256 kvm_pfn_t old_pfn; 257 gpa_t old_gpa; 258 259 write_lock_irq(&gpc->lock); 260 261 gpc->valid = false; 262 263 old_khva = gpc->khva - offset_in_page(gpc->khva); 264 old_gpa = gpc->gpa; 265 old_pfn = gpc->pfn; 266 267 /* 268 * We can leave the GPA → uHVA map cache intact but the PFN 269 * lookup will need to be redone even for the same page. 270 */ 271 gpc->khva = NULL; 272 gpc->pfn = KVM_PFN_ERR_FAULT; 273 274 write_unlock_irq(&gpc->lock); 275 276 __release_gpc(kvm, old_pfn, old_khva, old_gpa); 277 } 278 EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_unmap); 279 280 281 int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, 282 struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, 283 gpa_t gpa, unsigned long len) 284 { 285 WARN_ON_ONCE(!usage || (usage & KVM_GUEST_AND_HOST_USE_PFN) != usage); 286 287 if (!gpc->active) { 288 rwlock_init(&gpc->lock); 289 290 gpc->khva = NULL; 291 gpc->pfn = KVM_PFN_ERR_FAULT; 292 gpc->uhva = KVM_HVA_ERR_BAD; 293 gpc->vcpu = vcpu; 294 gpc->usage = usage; 295 gpc->valid = false; 296 gpc->active = true; 297 298 spin_lock(&kvm->gpc_lock); 299 list_add(&gpc->list, &kvm->gpc_list); 300 spin_unlock(&kvm->gpc_lock); 301 } 302 return kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpa, len); 303 } 304 EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_init); 305 306 void kvm_gfn_to_pfn_cache_destroy(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) 307 { 308 if (gpc->active) { 309 spin_lock(&kvm->gpc_lock); 310 list_del(&gpc->list); 311 spin_unlock(&kvm->gpc_lock); 312 313 kvm_gfn_to_pfn_cache_unmap(kvm, gpc); 314 gpc->active = false; 315 } 316 } 317 EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_destroy); 318