1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_svm.h" 7 #include "xe_userptr.h" 8 9 #include <linux/mm.h> 10 11 #include "xe_tlb_inval.h" 12 #include "xe_trace_bo.h" 13 14 static void xe_userptr_assert_in_notifier(struct xe_vm *vm) 15 { 16 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || 17 (lockdep_is_held(&vm->lock) && 18 lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && 19 dma_resv_held(xe_vm_resv(vm)))); 20 } 21 22 /** 23 * xe_vma_userptr_check_repin() - Advisory check for repin needed 24 * @uvma: The userptr vma 25 * 26 * Check if the userptr vma has been invalidated since last successful 27 * repin. The check is advisory only and can the function can be called 28 * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the 29 * vma userptr will remain valid after a lockless check, so typically 30 * the call needs to be followed by a proper check under the notifier_lock. 31 * 32 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. 33 */ 34 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) 35 { 36 return mmu_interval_check_retry(&uvma->userptr.notifier, 37 uvma->userptr.pages.notifier_seq) ? 38 -EAGAIN : 0; 39 } 40 41 /** 42 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs 43 * that need repinning. 44 * @vm: The VM. 45 * 46 * This function checks for whether the VM has userptrs that need repinning, 47 * and provides a release-type barrier on the svm.gpusvm.notifier_lock after 48 * checking. 49 * 50 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. 51 */ 52 int __xe_vm_userptr_needs_repin(struct xe_vm *vm) 53 { 54 lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock); 55 56 return (list_empty(&vm->userptr.repin_list) && 57 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 58 } 59 60 #if IS_ENABLED(CONFIG_PROVE_LOCKING) 61 static bool __xe_vma_userptr_lockdep(struct xe_userptr_vma *uvma) 62 { 63 struct xe_vma *vma = &uvma->vma; 64 struct xe_vm *vm = xe_vma_vm(vma); 65 66 return lockdep_is_held_type(&vm->lock, 0) || 67 (lockdep_is_held_type(&vm->lock, 1) && 68 lockdep_is_held_type(&vma->fault_lock, 0)); 69 } 70 71 #define xe_vma_userptr_lockdep(uvma) \ 72 lockdep_assert(__xe_vma_userptr_lockdep(uvma)) 73 #else 74 #define xe_vma_userptr_lockdep(uvma) 75 #endif 76 77 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) 78 { 79 struct xe_vma *vma = &uvma->vma; 80 struct xe_vm *vm = xe_vma_vm(vma); 81 struct xe_device *xe = vm->xe; 82 struct drm_gpusvm_ctx ctx = { 83 .read_only = xe_vma_read_only(vma), 84 .device_private_page_owner = xe_svm_private_page_owner(vm, false), 85 .allow_mixed = true, 86 }; 87 88 xe_vma_userptr_lockdep(uvma); 89 xe_assert(xe, xe_vma_is_userptr(vma)); 90 91 if (vma->gpuva.flags & XE_VMA_DESTROYED) 92 return 0; 93 94 return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 1, 95 uvma->userptr.notifier.mm, 96 &uvma->userptr.notifier, 97 xe_vma_userptr(vma), 98 xe_vma_userptr(vma) + xe_vma_size(vma), 99 &ctx); 100 } 101 102 static struct mmu_interval_notifier_finish * 103 xe_vma_userptr_do_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma, bool is_deferred) 104 { 105 struct xe_userptr *userptr = &uvma->userptr; 106 struct xe_vma *vma = &uvma->vma; 107 struct drm_gpusvm_ctx ctx = { 108 .in_notifier = true, 109 .read_only = xe_vma_read_only(vma), 110 }; 111 long err; 112 113 xe_userptr_assert_in_notifier(vm); 114 if (is_deferred) 115 xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted); 116 117 err = dma_resv_wait_timeout(xe_vm_resv(vm), 118 DMA_RESV_USAGE_BOOKKEEP, 119 false, MAX_SCHEDULE_TIMEOUT); 120 XE_WARN_ON(err <= 0); 121 122 if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) { 123 if (!userptr->finish_inuse) { 124 /* 125 * Defer the TLB wait to an extra pass so the caller 126 * can pipeline TLB flushes across GPUs before waiting 127 * on any of them. 128 */ 129 xe_assert(vm->xe, !userptr->tlb_inval_submitted); 130 userptr->finish_inuse = true; 131 userptr->tlb_inval_submitted = true; 132 err = xe_vm_invalidate_vma_submit(vma, &userptr->inval_batch); 133 XE_WARN_ON(err); 134 return &userptr->finish; 135 } 136 err = xe_vm_invalidate_vma(vma); 137 XE_WARN_ON(err); 138 } 139 140 if (is_deferred) 141 userptr->finish_inuse = false; 142 drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 143 xe_vma_size(vma) >> PAGE_SHIFT, &ctx); 144 return NULL; 145 } 146 147 static void 148 xe_vma_userptr_complete_tlb_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma) 149 { 150 struct xe_userptr *userptr = &uvma->userptr; 151 struct xe_vma *vma = &uvma->vma; 152 struct drm_gpusvm_ctx ctx = { 153 .in_notifier = true, 154 .read_only = xe_vma_read_only(vma), 155 }; 156 157 xe_userptr_assert_in_notifier(vm); 158 xe_assert(vm->xe, userptr->finish_inuse); 159 xe_assert(vm->xe, userptr->tlb_inval_submitted); 160 161 xe_tlb_inval_batch_wait(&userptr->inval_batch); 162 userptr->tlb_inval_submitted = false; 163 userptr->finish_inuse = false; 164 drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 165 xe_vma_size(vma) >> PAGE_SHIFT, &ctx); 166 } 167 168 static struct mmu_interval_notifier_finish * 169 xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma) 170 { 171 struct xe_userptr *userptr = &uvma->userptr; 172 struct xe_vma *vma = &uvma->vma; 173 struct dma_resv_iter cursor; 174 struct dma_fence *fence; 175 bool signaled = true; 176 177 xe_userptr_assert_in_notifier(vm); 178 179 /* 180 * Tell exec and rebind worker they need to repin and rebind this 181 * userptr. 182 */ 183 if (!xe_vm_in_fault_mode(vm) && 184 !(vma->gpuva.flags & XE_VMA_DESTROYED)) { 185 spin_lock(&vm->userptr.invalidated_lock); 186 list_move_tail(&userptr->invalidate_link, 187 &vm->userptr.invalidated); 188 spin_unlock(&vm->userptr.invalidated_lock); 189 } 190 191 /* 192 * Preempt fences turn into schedule disables, pipeline these. 193 * Note that even in fault mode, we need to wait for binds and 194 * unbinds to complete, and those are attached as BOOKMARK fences 195 * to the vm. 196 */ 197 dma_resv_iter_begin(&cursor, xe_vm_resv(vm), 198 DMA_RESV_USAGE_BOOKKEEP); 199 dma_resv_for_each_fence_unlocked(&cursor, fence) { 200 dma_fence_enable_signaling(fence); 201 if (signaled && !dma_fence_is_signaled(fence)) 202 signaled = false; 203 } 204 dma_resv_iter_end(&cursor); 205 206 /* 207 * Only one caller at a time can use the multi-pass state. 208 * If it's already in use, or all fences are already signaled, 209 * proceed directly to invalidation without deferring. 210 */ 211 if (signaled || userptr->finish_inuse) 212 return xe_vma_userptr_do_inval(vm, uvma, false); 213 214 /* Defer: the notifier core will call invalidate_finish once done. */ 215 userptr->finish_inuse = true; 216 217 return &userptr->finish; 218 } 219 220 static bool xe_vma_userptr_invalidate_start(struct mmu_interval_notifier *mni, 221 const struct mmu_notifier_range *range, 222 unsigned long cur_seq, 223 struct mmu_interval_notifier_finish **p_finish) 224 { 225 struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier); 226 struct xe_vma *vma = &uvma->vma; 227 struct xe_vm *vm = xe_vma_vm(vma); 228 229 xe_assert(vm->xe, xe_vma_is_userptr(vma)); 230 trace_xe_vma_userptr_invalidate(vma); 231 232 if (!mmu_notifier_range_blockable(range)) 233 return false; 234 235 vm_dbg(&xe_vma_vm(vma)->xe->drm, 236 "NOTIFIER PASS1: addr=0x%016llx, range=0x%016llx", 237 xe_vma_start(vma), xe_vma_size(vma)); 238 239 down_write(&vm->svm.gpusvm.notifier_lock); 240 mmu_interval_set_seq(mni, cur_seq); 241 242 *p_finish = xe_vma_userptr_invalidate_pass1(vm, uvma); 243 244 up_write(&vm->svm.gpusvm.notifier_lock); 245 if (!*p_finish) 246 trace_xe_vma_userptr_invalidate_complete(vma); 247 248 return true; 249 } 250 251 static void xe_vma_userptr_invalidate_finish(struct mmu_interval_notifier_finish *finish) 252 { 253 struct xe_userptr_vma *uvma = container_of(finish, typeof(*uvma), userptr.finish); 254 struct xe_vma *vma = &uvma->vma; 255 struct xe_vm *vm = xe_vma_vm(vma); 256 257 vm_dbg(&xe_vma_vm(vma)->xe->drm, 258 "NOTIFIER PASS2: addr=0x%016llx, range=0x%016llx", 259 xe_vma_start(vma), xe_vma_size(vma)); 260 261 down_write(&vm->svm.gpusvm.notifier_lock); 262 /* 263 * If a TLB invalidation was previously submitted (deferred from the 264 * synchronous pass1 fallback), wait for it and unmap pages. 265 * Otherwise, fences have now completed: invalidate the TLB and unmap. 266 */ 267 if (uvma->userptr.tlb_inval_submitted) 268 xe_vma_userptr_complete_tlb_inval(vm, uvma); 269 else 270 xe_vma_userptr_do_inval(vm, uvma, true); 271 up_write(&vm->svm.gpusvm.notifier_lock); 272 trace_xe_vma_userptr_invalidate_complete(vma); 273 } 274 275 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { 276 .invalidate_start = xe_vma_userptr_invalidate_start, 277 .invalidate_finish = xe_vma_userptr_invalidate_finish, 278 }; 279 280 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) 281 /** 282 * xe_vma_userptr_force_invalidate() - force invalidate a userptr 283 * @uvma: The userptr vma to invalidate 284 * 285 * Perform a forced userptr invalidation for testing purposes. 286 */ 287 void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) 288 { 289 struct mmu_interval_notifier_finish *finish; 290 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 291 292 /* Protect against concurrent userptr pinning */ 293 lockdep_assert_held(&vm->lock); 294 /* Protect against concurrent notifiers */ 295 lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); 296 /* 297 * Protect against concurrent instances of this function and 298 * the critical exec sections 299 */ 300 xe_vm_assert_held(vm); 301 302 if (!mmu_interval_read_retry(&uvma->userptr.notifier, 303 uvma->userptr.pages.notifier_seq)) 304 uvma->userptr.pages.notifier_seq -= 2; 305 306 finish = xe_vma_userptr_invalidate_pass1(vm, uvma); 307 if (finish) 308 finish = xe_vma_userptr_do_inval(vm, uvma, true); 309 if (finish) 310 xe_vma_userptr_complete_tlb_inval(vm, uvma); 311 } 312 #endif 313 314 int xe_vm_userptr_pin(struct xe_vm *vm) 315 { 316 struct xe_userptr_vma *uvma, *next; 317 int err = 0; 318 319 xe_assert(vm->xe, !xe_vm_in_fault_mode(vm)); 320 lockdep_assert_held_write(&vm->lock); 321 322 /* Collect invalidated userptrs */ 323 spin_lock(&vm->userptr.invalidated_lock); 324 xe_assert(vm->xe, list_empty(&vm->userptr.repin_list)); 325 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, 326 userptr.invalidate_link) { 327 list_del_init(&uvma->userptr.invalidate_link); 328 list_add_tail(&uvma->userptr.repin_link, 329 &vm->userptr.repin_list); 330 } 331 spin_unlock(&vm->userptr.invalidated_lock); 332 333 /* Pin and move to bind list */ 334 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 335 userptr.repin_link) { 336 err = xe_vma_userptr_pin_pages(uvma); 337 if (err == -EFAULT) { 338 list_del_init(&uvma->userptr.repin_link); 339 /* 340 * We might have already done the pin once already, but 341 * then had to retry before the re-bind happened, due 342 * some other condition in the caller, but in the 343 * meantime the userptr got dinged by the notifier such 344 * that we need to revalidate here, but this time we hit 345 * the EFAULT. In such a case make sure we remove 346 * ourselves from the rebind list to avoid going down in 347 * flames. 348 */ 349 if (!list_empty(&uvma->vma.combined_links.rebind)) 350 list_del_init(&uvma->vma.combined_links.rebind); 351 352 /* Wait for pending binds */ 353 xe_vm_lock(vm, false); 354 dma_resv_wait_timeout(xe_vm_resv(vm), 355 DMA_RESV_USAGE_BOOKKEEP, 356 false, MAX_SCHEDULE_TIMEOUT); 357 358 down_read(&vm->svm.gpusvm.notifier_lock); 359 err = xe_vm_invalidate_vma(&uvma->vma); 360 up_read(&vm->svm.gpusvm.notifier_lock); 361 xe_vm_unlock(vm); 362 if (err) 363 break; 364 } else { 365 if (err) 366 break; 367 368 list_del_init(&uvma->userptr.repin_link); 369 list_move_tail(&uvma->vma.combined_links.rebind, 370 &vm->rebind_list); 371 } 372 } 373 374 if (err) { 375 down_write(&vm->svm.gpusvm.notifier_lock); 376 spin_lock(&vm->userptr.invalidated_lock); 377 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, 378 userptr.repin_link) { 379 list_del_init(&uvma->userptr.repin_link); 380 list_move_tail(&uvma->userptr.invalidate_link, 381 &vm->userptr.invalidated); 382 } 383 spin_unlock(&vm->userptr.invalidated_lock); 384 up_write(&vm->svm.gpusvm.notifier_lock); 385 } 386 return err; 387 } 388 389 /** 390 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs 391 * that need repinning. 392 * @vm: The VM. 393 * 394 * This function does an advisory check for whether the VM has userptrs that 395 * need repinning. 396 * 397 * Return: 0 if there are no indications of userptrs needing repinning, 398 * -EAGAIN if there are. 399 */ 400 int xe_vm_userptr_check_repin(struct xe_vm *vm) 401 { 402 return (list_empty_careful(&vm->userptr.repin_list) && 403 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; 404 } 405 406 int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start, 407 unsigned long range) 408 { 409 struct xe_userptr *userptr = &uvma->userptr; 410 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 411 int err; 412 413 INIT_LIST_HEAD(&userptr->invalidate_link); 414 INIT_LIST_HEAD(&userptr->repin_link); 415 416 drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm); 417 418 err = mmu_interval_notifier_insert(&userptr->notifier, current->mm, 419 start, range, 420 &vma_userptr_notifier_ops); 421 if (err) 422 return err; 423 424 return 0; 425 } 426 427 void xe_userptr_remove(struct xe_userptr_vma *uvma) 428 { 429 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 430 struct xe_userptr *userptr = &uvma->userptr; 431 432 drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 433 xe_vma_size(&uvma->vma) >> PAGE_SHIFT); 434 435 /* 436 * Since userptr pages are not pinned, we can't remove 437 * the notifier until we're sure the GPU is not accessing 438 * them anymore 439 */ 440 mmu_interval_notifier_remove(&userptr->notifier); 441 } 442 443 void xe_userptr_destroy(struct xe_userptr_vma *uvma) 444 { 445 struct xe_vm *vm = xe_vma_vm(&uvma->vma); 446 447 spin_lock(&vm->userptr.invalidated_lock); 448 xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link)); 449 list_del(&uvma->userptr.invalidate_link); 450 spin_unlock(&vm->userptr.invalidated_lock); 451 } 452