1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_svm.h" 7 #include "xe_userptr.h" 8 9 #include <linux/mm.h> 10 11 #include "xe_trace_bo.h" 12 13 /** 14 * xe_vma_userptr_check_repin() - Advisory check for repin needed 15 * @uvma: The userptr vma 16 * 17 * Check if the userptr vma has been invalidated since last successful 18 * repin. The check is advisory only and can the function can be called 19 * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the 20 * vma userptr will remain valid after a lockless check, so typically 21 * the call needs to be followed by a proper check under the notifier_lock. 22 * 23 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 24 */ 25 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) 26 { 27 return mmu_interval_check_retry(&uvma->userptr.notifier, 28 uvma->userptr.pages.notifier_seq) ? 29 -EAGAIN : 0; 30 } 31 32 /** 33 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 34 * that need repinning. 35 * @vm: The VM. 36 * 37 * This function checks for whether the VM has userptrs that need repinning, 38 * and provides a release-type barrier on the svm.gpusvm.notifier_lock after 39 * checking. 40 * 41 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 42 */ 43 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 44 { 45 lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock); 46 47 return (list_empty(&vm->userptr.repin_list) && 48 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 49 } 50 51 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) 52 { 53 struct xe_vma *vma = &uvma->vma; 54 struct xe_vm *vm = xe_vma_vm(vma); 55 struct xe_device *xe = vm->xe; 56 struct drm_gpusvm_ctx ctx = { 57 .read_only = xe_vma_read_only(vma), 58 .device_private_page_owner = xe_svm_devm_owner(xe), 59 .allow_mixed = true, 60 }; 61 62 lockdep_assert_held(&vm->lock); 63 xe_assert(xe, xe_vma_is_userptr(vma)); 64 65 if (vma->gpuva.flags & XE_VMA_DESTROYED) 66 return 0; 67 68 return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 69 uvma->userptr.notifier.mm, 70 &uvma->userptr.notifier, 71 xe_vma_userptr(vma), 72 xe_vma_userptr(vma) + xe_vma_size(vma), 73 &ctx); 74 } 75 76 static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma) 77 { 78 struct xe_userptr *userptr = &uvma->userptr; 79 struct xe_vma *vma = &uvma->vma; 80 struct dma_resv_iter cursor; 81 struct dma_fence *fence; 82 struct drm_gpusvm_ctx ctx = { 83 .in_notifier = true, 84 .read_only = xe_vma_read_only(vma), 85 }; 86 long err; 87 88 /* 89 * Tell exec and rebind worker they need to repin and rebind this 90 * userptr. 91 */ 92 if (!xe_vm_in_fault_mode(vm) && 93 !(vma->gpuva.flags & XE_VMA_DESTROYED)) { 94 spin_lock(&vm->userptr.invalidated_lock); 95 list_move_tail(&userptr->invalidate_link, 96 &vm->userptr.invalidated); 97 spin_unlock(&vm->userptr.invalidated_lock); 98 } 99 100 /* 101 * Preempt fences turn into schedule disables, pipeline these. 102 * Note that even in fault mode, we need to wait for binds and 103 * unbinds to complete, and those are attached as BOOKMARK fences 104 * to the vm. 105 */ 106 dma_resv_iter_begin(&cursor, xe_vm_resv(vm), 107 DMA_RESV_USAGE_BOOKKEEP); 108 dma_resv_for_each_fence_unlocked(&cursor, fence) 109 dma_fence_enable_sw_signaling(fence); 110 dma_resv_iter_end(&cursor); 111 112 err = dma_resv_wait_timeout(xe_vm_resv(vm), 113 DMA_RESV_USAGE_BOOKKEEP, 114 false, MAX_SCHEDULE_TIMEOUT); 115 XE_WARN_ON(err <= 0); 116 117 if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) { 118 err = xe_vm_invalidate_vma(vma); 119 XE_WARN_ON(err); 120 } 121 122 drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 123 xe_vma_size(vma) >> PAGE_SHIFT, &ctx); 124 } 125 126 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, 127 const struct mmu_notifier_range *range, 128 unsigned long cur_seq) 129 { 130 struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier); 131 struct xe_vma *vma = &uvma->vma; 132 struct xe_vm *vm = xe_vma_vm(vma); 133 134 xe_assert(vm->xe, xe_vma_is_userptr(vma)); 135 trace_xe_vma_userptr_invalidate(vma); 136 137 if (!mmu_notifier_range_blockable(range)) 138 return false; 139 140 vm_dbg(&xe_vma_vm(vma)->xe->drm, 141 "NOTIFIER: addr=0x%016llx, range=0x%016llx", 142 xe_vma_start(vma), xe_vma_size(vma)); 143 144 down_write(&vm->svm.gpusvm.notifier_lock); 145 mmu_interval_set_seq(mni, cur_seq); 146 147 __vma_userptr_invalidate(vm, uvma); 148 up_write(&vm->svm.gpusvm.notifier_lock); 149 trace_xe_vma_userptr_invalidate_complete(vma); 150 151 return true; 152 } 153 154 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 155 .invalidate = vma_userptr_invalidate, 156 }; 157 158 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 159 /** 160 * xe_vma_userptr_force_invalidate() - force invalidate a userptr 161 * @uvma: The userptr vma to invalidate 162 * 163 * Perform a forced userptr invalidation for testing purposes. 164 */ 165 void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) 166 { 167 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 168 169 /* Protect against concurrent userptr pinning */ 170 lockdep_assert_held(&vm->lock); 171 /* Protect against concurrent notifiers */ 172 lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); 173 /* 174 * Protect against concurrent instances of this function and 175 * the critical exec sections 176 */ 177 xe_vm_assert_held(vm); 178 179 if (!mmu_interval_read_retry(&uvma->userptr.notifier, 180 uvma->userptr.pages.notifier_seq)) 181 uvma->userptr.pages.notifier_seq -= 2; 182 __vma_userptr_invalidate(vm, uvma); 183 } 184 #endif 185 186 int xe_vm_userptr_pin(struct xe_vm *vm) 187 { 188 struct xe_userptr_vma *uvma, *next; 189 int err = 0; 190 191 xe_assert(vm->xe, !xe_vm_in_fault_mode(vm)); 192 lockdep_assert_held_write(&vm->lock); 193 194 /* Collect invalidated userptrs */ 195 spin_lock(&vm->userptr.invalidated_lock); 196 xe_assert(vm->xe, list_empty(&vm->userptr.repin_list)); 197 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, 198 userptr.invalidate_link) { 199 list_del_init(&uvma->userptr.invalidate_link); 200 list_add_tail(&uvma->userptr.repin_link, 201 &vm->userptr.repin_list); 202 } 203 spin_unlock(&vm->userptr.invalidated_lock); 204 205 /* Pin and move to bind list */ 206 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 207 userptr.repin_link) { 208 err = xe_vma_userptr_pin_pages(uvma); 209 if (err == -EFAULT) { 210 list_del_init(&uvma->userptr.repin_link); 211 /* 212 * We might have already done the pin once already, but 213 * then had to retry before the re-bind happened, due 214 * some other condition in the caller, but in the 215 * meantime the userptr got dinged by the notifier such 216 * that we need to revalidate here, but this time we hit 217 * the EFAULT. In such a case make sure we remove 218 * ourselves from the rebind list to avoid going down in 219 * flames. 220 */ 221 if (!list_empty(&uvma->vma.combined_links.rebind)) 222 list_del_init(&uvma->vma.combined_links.rebind); 223 224 /* Wait for pending binds */ 225 xe_vm_lock(vm, false); 226 dma_resv_wait_timeout(xe_vm_resv(vm), 227 DMA_RESV_USAGE_BOOKKEEP, 228 false, MAX_SCHEDULE_TIMEOUT); 229 230 down_read(&vm->svm.gpusvm.notifier_lock); 231 err = xe_vm_invalidate_vma(&uvma->vma); 232 up_read(&vm->svm.gpusvm.notifier_lock); 233 xe_vm_unlock(vm); 234 if (err) 235 break; 236 } else { 237 if (err) 238 break; 239 240 list_del_init(&uvma->userptr.repin_link); 241 list_move_tail(&uvma->vma.combined_links.rebind, 242 &vm->rebind_list); 243 } 244 } 245 246 if (err) { 247 down_write(&vm->svm.gpusvm.notifier_lock); 248 spin_lock(&vm->userptr.invalidated_lock); 249 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 250 userptr.repin_link) { 251 list_del_init(&uvma->userptr.repin_link); 252 list_move_tail(&uvma->userptr.invalidate_link, 253 &vm->userptr.invalidated); 254 } 255 spin_unlock(&vm->userptr.invalidated_lock); 256 up_write(&vm->svm.gpusvm.notifier_lock); 257 } 258 return err; 259 } 260 261 /** 262 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 263 * that need repinning. 264 * @vm: The VM. 265 * 266 * This function does an advisory check for whether the VM has userptrs that 267 * need repinning. 268 * 269 * Return: 0 if there are no indications of userptrs needing repinning, 270 * -EAGAIN if there are. 271 */ 272 int xe_vm_userptr_check_repin(struct xe_vm *vm) 273 { 274 return (list_empty_careful(&vm->userptr.repin_list) && 275 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 276 } 277 278 int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start, 279 unsigned long range) 280 { 281 struct xe_userptr *userptr = &uvma->userptr; 282 int err; 283 284 INIT_LIST_HEAD(&userptr->invalidate_link); 285 INIT_LIST_HEAD(&userptr->repin_link); 286 287 err = mmu_interval_notifier_insert(&userptr->notifier, current->mm, 288 start, range, 289 &vma_userptr_notifier_ops); 290 if (err) 291 return err; 292 293 userptr->pages.notifier_seq = LONG_MAX; 294 295 return 0; 296 } 297 298 void xe_userptr_remove(struct xe_userptr_vma *uvma) 299 { 300 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 301 struct xe_userptr *userptr = &uvma->userptr; 302 303 drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 304 xe_vma_size(&uvma->vma) >> PAGE_SHIFT); 305 306 /* 307 * Since userptr pages are not pinned, we can't remove 308 * the notifier until we're sure the GPU is not accessing 309 * them anymore 310 */ 311 mmu_interval_notifier_remove(&userptr->notifier); 312 } 313 314 void xe_userptr_destroy(struct xe_userptr_vma *uvma) 315 { 316 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 317 318 spin_lock(&vm->userptr.invalidated_lock); 319 xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link)); 320 list_del(&uvma->userptr.invalidate_link); 321 spin_unlock(&vm->userptr.invalidated_lock); 322 } 323