18  * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the
32  * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
34  * @vm: The VM.
36  * This function checks for whether the VM has userptrs that need repinning,
42 int __xe_vm_userptr_needs_repin(struct xe_vm *vm)  in __xe_vm_userptr_needs_repin()  argument
44 	lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock);  in __xe_vm_userptr_needs_repin()
46 	return (list_empty(&vm->userptr.repin_list) &&  in __xe_vm_userptr_needs_repin()
47 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in __xe_vm_userptr_needs_repin()
53 	struct xe_vm *vm = xe_vma_vm(vma);  in xe_vma_userptr_pin_pages()  local
54 	struct xe_device *xe = vm->xe;  in xe_vma_userptr_pin_pages()
60 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_pin_pages()
66 	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_pin_pages()
74 static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma)  in __vma_userptr_invalidate()  argument
90 	if (!xe_vm_in_fault_mode(vm) &&  in __vma_userptr_invalidate()
92 		spin_lock(&vm->userptr.invalidated_lock);  in __vma_userptr_invalidate()
94 			       &vm->userptr.invalidated);  in __vma_userptr_invalidate()
95 		spin_unlock(&vm->userptr.invalidated_lock);  in __vma_userptr_invalidate()
102 	 * to the vm.  in __vma_userptr_invalidate()
104 	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),  in __vma_userptr_invalidate()
110 	err = dma_resv_wait_timeout(xe_vm_resv(vm),  in __vma_userptr_invalidate()
115 	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {  in __vma_userptr_invalidate()
120 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in __vma_userptr_invalidate()
130 	struct xe_vm *vm = xe_vma_vm(vma);  in vma_userptr_invalidate()  local
132 	xe_assert(vm->xe, xe_vma_is_userptr(vma));  in vma_userptr_invalidate()
142 	down_write(&vm->svm.gpusvm.notifier_lock);  in vma_userptr_invalidate()
145 	__vma_userptr_invalidate(vm, uvma);  in vma_userptr_invalidate()
146 	up_write(&vm->svm.gpusvm.notifier_lock);  in vma_userptr_invalidate()
165 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_vma_userptr_force_invalidate()  local
168 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_force_invalidate()
170 	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_force_invalidate()
175 	xe_vm_assert_held(vm);  in xe_vma_userptr_force_invalidate()
180 	__vma_userptr_invalidate(vm, uvma);  in xe_vma_userptr_force_invalidate()
184 int xe_vm_userptr_pin(struct xe_vm *vm)  in xe_vm_userptr_pin()  argument
189 	xe_assert(vm->xe, !xe_vm_in_fault_mode(vm));  in xe_vm_userptr_pin()
190 	lockdep_assert_held_write(&vm->lock);  in xe_vm_userptr_pin()
193 	spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
194 	xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));  in xe_vm_userptr_pin()
195 	list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,  in xe_vm_userptr_pin()
199 			      &vm->userptr.repin_list);  in xe_vm_userptr_pin()
201 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
204 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
223 			xe_vm_lock(vm, false);  in xe_vm_userptr_pin()
224 			dma_resv_wait_timeout(xe_vm_resv(vm),  in xe_vm_userptr_pin()
228 			down_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
230 			up_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
231 			xe_vm_unlock(vm);  in xe_vm_userptr_pin()
240 				       &vm->rebind_list);  in xe_vm_userptr_pin()
245 		down_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
246 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
247 		list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
251 				       &vm->userptr.invalidated);  in xe_vm_userptr_pin()
253 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
254 		up_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
260  * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
262  * @vm: The VM.
264  * This function does an advisory check for whether the VM has userptrs that
270 int xe_vm_userptr_check_repin(struct xe_vm *vm)  in xe_vm_userptr_check_repin()  argument
272 	return (list_empty_careful(&vm->userptr.repin_list) &&  in xe_vm_userptr_check_repin()
273 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in xe_vm_userptr_check_repin()
298 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_remove()  local
301 	drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_userptr_remove()
314 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_destroy()  local
316 	spin_lock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()
317 	xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));  in xe_userptr_destroy()
319 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()