1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_userptr.h" 7 8 #include <linux/mm.h> 9 10 #include "xe_trace_bo.h" 11 12 /** 13 * xe_vma_userptr_check_repin() - Advisory check for repin needed 14 * @uvma: The userptr vma 15 * 16 * Check if the userptr vma has been invalidated since last successful 17 * repin. The check is advisory only and can the function can be called 18 * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the 19 * vma userptr will remain valid after a lockless check, so typically 20 * the call needs to be followed by a proper check under the notifier_lock. 21 * 22 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 23 */ 24 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) 25 { 26 return mmu_interval_check_retry(&uvma->userptr.notifier, 27 uvma->userptr.pages.notifier_seq) ? 28 -EAGAIN : 0; 29 } 30 31 /** 32 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 33 * that need repinning. 34 * @vm: The VM. 35 * 36 * This function checks for whether the VM has userptrs that need repinning, 37 * and provides a release-type barrier on the svm.gpusvm.notifier_lock after 38 * checking. 39 * 40 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 41 */ 42 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 43 { 44 lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock); 45 46 return (list_empty(&vm->userptr.repin_list) && 47 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 48 } 49 50 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) 51 { 52 struct xe_vma *vma = &uvma->vma; 53 struct xe_vm *vm = xe_vma_vm(vma); 54 struct xe_device *xe = vm->xe; 55 struct drm_gpusvm_ctx ctx = { 56 .read_only = xe_vma_read_only(vma), 57 }; 58 59 lockdep_assert_held(&vm->lock); 60 xe_assert(xe, xe_vma_is_userptr(vma)); 61 62 if (vma->gpuva.flags & XE_VMA_DESTROYED) 63 return 0; 64 65 return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 66 uvma->userptr.notifier.mm, 67 &uvma->userptr.notifier, 68 xe_vma_userptr(vma), 69 xe_vma_userptr(vma) + xe_vma_size(vma), 70 &ctx); 71 } 72 73 static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma) 74 { 75 struct xe_userptr *userptr = &uvma->userptr; 76 struct xe_vma *vma = &uvma->vma; 77 struct dma_resv_iter cursor; 78 struct dma_fence *fence; 79 struct drm_gpusvm_ctx ctx = { 80 .in_notifier = true, 81 .read_only = xe_vma_read_only(vma), 82 }; 83 long err; 84 85 /* 86 * Tell exec and rebind worker they need to repin and rebind this 87 * userptr. 88 */ 89 if (!xe_vm_in_fault_mode(vm) && 90 !(vma->gpuva.flags & XE_VMA_DESTROYED)) { 91 spin_lock(&vm->userptr.invalidated_lock); 92 list_move_tail(&userptr->invalidate_link, 93 &vm->userptr.invalidated); 94 spin_unlock(&vm->userptr.invalidated_lock); 95 } 96 97 /* 98 * Preempt fences turn into schedule disables, pipeline these. 99 * Note that even in fault mode, we need to wait for binds and 100 * unbinds to complete, and those are attached as BOOKMARK fences 101 * to the vm. 102 */ 103 dma_resv_iter_begin(&cursor, xe_vm_resv(vm), 104 DMA_RESV_USAGE_BOOKKEEP); 105 dma_resv_for_each_fence_unlocked(&cursor, fence) 106 dma_fence_enable_sw_signaling(fence); 107 dma_resv_iter_end(&cursor); 108 109 err = dma_resv_wait_timeout(xe_vm_resv(vm), 110 DMA_RESV_USAGE_BOOKKEEP, 111 false, MAX_SCHEDULE_TIMEOUT); 112 XE_WARN_ON(err <= 0); 113 114 if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) { 115 err = xe_vm_invalidate_vma(vma); 116 XE_WARN_ON(err); 117 } 118 119 drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 120 xe_vma_size(vma) >> PAGE_SHIFT, &ctx); 121 } 122 123 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, 124 const struct mmu_notifier_range *range, 125 unsigned long cur_seq) 126 { 127 struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier); 128 struct xe_vma *vma = &uvma->vma; 129 struct xe_vm *vm = xe_vma_vm(vma); 130 131 xe_assert(vm->xe, xe_vma_is_userptr(vma)); 132 trace_xe_vma_userptr_invalidate(vma); 133 134 if (!mmu_notifier_range_blockable(range)) 135 return false; 136 137 vm_dbg(&xe_vma_vm(vma)->xe->drm, 138 "NOTIFIER: addr=0x%016llx, range=0x%016llx", 139 xe_vma_start(vma), xe_vma_size(vma)); 140 141 down_write(&vm->svm.gpusvm.notifier_lock); 142 mmu_interval_set_seq(mni, cur_seq); 143 144 __vma_userptr_invalidate(vm, uvma); 145 up_write(&vm->svm.gpusvm.notifier_lock); 146 trace_xe_vma_userptr_invalidate_complete(vma); 147 148 return true; 149 } 150 151 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 152 .invalidate = vma_userptr_invalidate, 153 }; 154 155 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 156 /** 157 * xe_vma_userptr_force_invalidate() - force invalidate a userptr 158 * @uvma: The userptr vma to invalidate 159 * 160 * Perform a forced userptr invalidation for testing purposes. 161 */ 162 void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) 163 { 164 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 165 166 /* Protect against concurrent userptr pinning */ 167 lockdep_assert_held(&vm->lock); 168 /* Protect against concurrent notifiers */ 169 lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); 170 /* 171 * Protect against concurrent instances of this function and 172 * the critical exec sections 173 */ 174 xe_vm_assert_held(vm); 175 176 if (!mmu_interval_read_retry(&uvma->userptr.notifier, 177 uvma->userptr.pages.notifier_seq)) 178 uvma->userptr.pages.notifier_seq -= 2; 179 __vma_userptr_invalidate(vm, uvma); 180 } 181 #endif 182 183 int xe_vm_userptr_pin(struct xe_vm *vm) 184 { 185 struct xe_userptr_vma *uvma, *next; 186 int err = 0; 187 188 xe_assert(vm->xe, !xe_vm_in_fault_mode(vm)); 189 lockdep_assert_held_write(&vm->lock); 190 191 /* Collect invalidated userptrs */ 192 spin_lock(&vm->userptr.invalidated_lock); 193 xe_assert(vm->xe, list_empty(&vm->userptr.repin_list)); 194 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, 195 userptr.invalidate_link) { 196 list_del_init(&uvma->userptr.invalidate_link); 197 list_add_tail(&uvma->userptr.repin_link, 198 &vm->userptr.repin_list); 199 } 200 spin_unlock(&vm->userptr.invalidated_lock); 201 202 /* Pin and move to bind list */ 203 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 204 userptr.repin_link) { 205 err = xe_vma_userptr_pin_pages(uvma); 206 if (err == -EFAULT) { 207 list_del_init(&uvma->userptr.repin_link); 208 /* 209 * We might have already done the pin once already, but 210 * then had to retry before the re-bind happened, due 211 * some other condition in the caller, but in the 212 * meantime the userptr got dinged by the notifier such 213 * that we need to revalidate here, but this time we hit 214 * the EFAULT. In such a case make sure we remove 215 * ourselves from the rebind list to avoid going down in 216 * flames. 217 */ 218 if (!list_empty(&uvma->vma.combined_links.rebind)) 219 list_del_init(&uvma->vma.combined_links.rebind); 220 221 /* Wait for pending binds */ 222 xe_vm_lock(vm, false); 223 dma_resv_wait_timeout(xe_vm_resv(vm), 224 DMA_RESV_USAGE_BOOKKEEP, 225 false, MAX_SCHEDULE_TIMEOUT); 226 227 down_read(&vm->svm.gpusvm.notifier_lock); 228 err = xe_vm_invalidate_vma(&uvma->vma); 229 up_read(&vm->svm.gpusvm.notifier_lock); 230 xe_vm_unlock(vm); 231 if (err) 232 break; 233 } else { 234 if (err) 235 break; 236 237 list_del_init(&uvma->userptr.repin_link); 238 list_move_tail(&uvma->vma.combined_links.rebind, 239 &vm->rebind_list); 240 } 241 } 242 243 if (err) { 244 down_write(&vm->svm.gpusvm.notifier_lock); 245 spin_lock(&vm->userptr.invalidated_lock); 246 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 247 userptr.repin_link) { 248 list_del_init(&uvma->userptr.repin_link); 249 list_move_tail(&uvma->userptr.invalidate_link, 250 &vm->userptr.invalidated); 251 } 252 spin_unlock(&vm->userptr.invalidated_lock); 253 up_write(&vm->svm.gpusvm.notifier_lock); 254 } 255 return err; 256 } 257 258 /** 259 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 260 * that need repinning. 261 * @vm: The VM. 262 * 263 * This function does an advisory check for whether the VM has userptrs that 264 * need repinning. 265 * 266 * Return: 0 if there are no indications of userptrs needing repinning, 267 * -EAGAIN if there are. 268 */ 269 int xe_vm_userptr_check_repin(struct xe_vm *vm) 270 { 271 return (list_empty_careful(&vm->userptr.repin_list) && 272 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 273 } 274 275 int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start, 276 unsigned long range) 277 { 278 struct xe_userptr *userptr = &uvma->userptr; 279 int err; 280 281 INIT_LIST_HEAD(&userptr->invalidate_link); 282 INIT_LIST_HEAD(&userptr->repin_link); 283 284 err = mmu_interval_notifier_insert(&userptr->notifier, current->mm, 285 start, range, 286 &vma_userptr_notifier_ops); 287 if (err) 288 return err; 289 290 userptr->pages.notifier_seq = LONG_MAX; 291 292 return 0; 293 } 294 295 void xe_userptr_remove(struct xe_userptr_vma *uvma) 296 { 297 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 298 struct xe_userptr *userptr = &uvma->userptr; 299 300 drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 301 xe_vma_size(&uvma->vma) >> PAGE_SHIFT); 302 303 /* 304 * Since userptr pages are not pinned, we can't remove 305 * the notifier until we're sure the GPU is not accessing 306 * them anymore 307 */ 308 mmu_interval_notifier_remove(&userptr->notifier); 309 } 310 311 void xe_userptr_destroy(struct xe_userptr_vma *uvma) 312 { 313 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 314 315 spin_lock(&vm->userptr.invalidated_lock); 316 xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link)); 317 list_del(&uvma->userptr.invalidate_link); 318 spin_unlock(&vm->userptr.invalidated_lock); 319 } 320