14 static void xe_userptr_assert_in_notifier(struct xe_vm *vm)  in xe_userptr_assert_in_notifier()  argument
16 	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||  in xe_userptr_assert_in_notifier()
17 		       (lockdep_is_held(&vm->lock) &&  in xe_userptr_assert_in_notifier()
18 			lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&  in xe_userptr_assert_in_notifier()
19 			dma_resv_held(xe_vm_resv(vm))));  in xe_userptr_assert_in_notifier()
52 int __xe_vm_userptr_needs_repin(struct xe_vm *vm)  in __xe_vm_userptr_needs_repin()  argument
54 	lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock);  in __xe_vm_userptr_needs_repin()
56 	return (list_empty(&vm->userptr.repin_list) &&  in __xe_vm_userptr_needs_repin()
57 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in __xe_vm_userptr_needs_repin()
63 	struct xe_vm *vm = xe_vma_vm(vma);  in xe_vma_userptr_pin_pages()  local
64 	struct xe_device *xe = vm->xe;  in xe_vma_userptr_pin_pages()
67 		.device_private_page_owner = xe_svm_private_page_owner(vm, false),  in xe_vma_userptr_pin_pages()
71 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_pin_pages()
77 	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_pin_pages()
86 xe_vma_userptr_do_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma, bool is_deferred)  in xe_vma_userptr_do_inval()  argument
96 	xe_userptr_assert_in_notifier(vm);  in xe_vma_userptr_do_inval()
98 		xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted);  in xe_vma_userptr_do_inval()
100 	err = dma_resv_wait_timeout(xe_vm_resv(vm),  in xe_vma_userptr_do_inval()
105 	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {  in xe_vma_userptr_do_inval()
112 			xe_assert(vm->xe, !userptr->tlb_inval_submitted);  in xe_vma_userptr_do_inval()
125 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_do_inval()
131 xe_vma_userptr_complete_tlb_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma)  in xe_vma_userptr_complete_tlb_inval()  argument
140 	xe_userptr_assert_in_notifier(vm);  in xe_vma_userptr_complete_tlb_inval()
141 	xe_assert(vm->xe, userptr->finish_inuse);  in xe_vma_userptr_complete_tlb_inval()
142 	xe_assert(vm->xe, userptr->tlb_inval_submitted);  in xe_vma_userptr_complete_tlb_inval()
147 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_complete_tlb_inval()
152 xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma)  in xe_vma_userptr_invalidate_pass1()  argument
160 	xe_userptr_assert_in_notifier(vm);  in xe_vma_userptr_invalidate_pass1()
166 	if (!xe_vm_in_fault_mode(vm) &&  in xe_vma_userptr_invalidate_pass1()
168 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vma_userptr_invalidate_pass1()
170 			       &vm->userptr.invalidated);  in xe_vma_userptr_invalidate_pass1()
171 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vma_userptr_invalidate_pass1()
180 	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),  in xe_vma_userptr_invalidate_pass1()
195 		return xe_vma_userptr_do_inval(vm, uvma, false);  in xe_vma_userptr_invalidate_pass1()
210 	struct xe_vm *vm = xe_vma_vm(vma);  in xe_vma_userptr_invalidate_start()  local
212 	xe_assert(vm->xe, xe_vma_is_userptr(vma));  in xe_vma_userptr_invalidate_start()
222 	down_write(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_invalidate_start()
225 	*p_finish = xe_vma_userptr_invalidate_pass1(vm, uvma);  in xe_vma_userptr_invalidate_start()
227 	up_write(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_invalidate_start()
238 	struct xe_vm *vm = xe_vma_vm(vma);  in xe_vma_userptr_invalidate_finish()  local
244 	down_write(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_invalidate_finish()
251 		xe_vma_userptr_complete_tlb_inval(vm, uvma);  in xe_vma_userptr_invalidate_finish()
253 		xe_vma_userptr_do_inval(vm, uvma, true);  in xe_vma_userptr_invalidate_finish()
254 	up_write(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_invalidate_finish()
273 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_vma_userptr_force_invalidate()  local
276 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_force_invalidate()
278 	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_force_invalidate()
283 	xe_vm_assert_held(vm);  in xe_vma_userptr_force_invalidate()
289 	finish = xe_vma_userptr_invalidate_pass1(vm, uvma);  in xe_vma_userptr_force_invalidate()
291 		finish = xe_vma_userptr_do_inval(vm, uvma, true);  in xe_vma_userptr_force_invalidate()
293 		xe_vma_userptr_complete_tlb_inval(vm, uvma);  in xe_vma_userptr_force_invalidate()
297 int xe_vm_userptr_pin(struct xe_vm *vm)  in xe_vm_userptr_pin()  argument
302 	xe_assert(vm->xe, !xe_vm_in_fault_mode(vm));  in xe_vm_userptr_pin()
303 	lockdep_assert_held_write(&vm->lock);  in xe_vm_userptr_pin()
306 	spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
307 	xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));  in xe_vm_userptr_pin()
308 	list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,  in xe_vm_userptr_pin()
312 			      &vm->userptr.repin_list);  in xe_vm_userptr_pin()
314 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
317 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
336 			xe_vm_lock(vm, false);  in xe_vm_userptr_pin()
337 			dma_resv_wait_timeout(xe_vm_resv(vm),  in xe_vm_userptr_pin()
341 			down_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
343 			up_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
344 			xe_vm_unlock(vm);  in xe_vm_userptr_pin()
353 				       &vm->rebind_list);  in xe_vm_userptr_pin()
358 		down_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
359 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
360 		list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
364 				       &vm->userptr.invalidated);  in xe_vm_userptr_pin()
366 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
367 		up_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
383 int xe_vm_userptr_check_repin(struct xe_vm *vm)  in xe_vm_userptr_check_repin()  argument
385 	return (list_empty_careful(&vm->userptr.repin_list) &&  in xe_vm_userptr_check_repin()
386 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in xe_vm_userptr_check_repin()
393 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_setup()  local
399 	drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm);  in xe_userptr_setup()
412 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_remove()  local
415 	drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_userptr_remove()
428 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_destroy()  local
430 	spin_lock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()
431 	xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));  in xe_userptr_destroy()
433 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()