36 	return mmu_interval_check_retry(&uvma->userptr.notifier,  in xe_vma_userptr_check_repin()
37 					uvma->userptr.pages.notifier_seq) ?  in xe_vma_userptr_check_repin()
56 	return (list_empty(&vm->userptr.repin_list) &&  in __xe_vm_userptr_needs_repin()
57 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in __xe_vm_userptr_needs_repin()
77 	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_pin_pages()
78 				    uvma->userptr.notifier.mm,  in xe_vma_userptr_pin_pages()
79 				    &uvma->userptr.notifier,  in xe_vma_userptr_pin_pages()
88 	struct xe_userptr *userptr = &uvma->userptr;  in xe_vma_userptr_do_inval()  local
98 		xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted);  in xe_vma_userptr_do_inval()
105 	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {  in xe_vma_userptr_do_inval()
106 		if (!userptr->finish_inuse) {  in xe_vma_userptr_do_inval()
112 			xe_assert(vm->xe, !userptr->tlb_inval_submitted);  in xe_vma_userptr_do_inval()
113 			userptr->finish_inuse = true;  in xe_vma_userptr_do_inval()
114 			userptr->tlb_inval_submitted = true;  in xe_vma_userptr_do_inval()
115 			err = xe_vm_invalidate_vma_submit(vma, &userptr->inval_batch);  in xe_vma_userptr_do_inval()
117 			return &userptr->finish;  in xe_vma_userptr_do_inval()
124 		userptr->finish_inuse = false;  in xe_vma_userptr_do_inval()
125 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_do_inval()
133 	struct xe_userptr *userptr = &uvma->userptr;  in xe_vma_userptr_complete_tlb_inval()  local
141 	xe_assert(vm->xe, userptr->finish_inuse);  in xe_vma_userptr_complete_tlb_inval()
142 	xe_assert(vm->xe, userptr->tlb_inval_submitted);  in xe_vma_userptr_complete_tlb_inval()
144 	xe_tlb_inval_batch_wait(&userptr->inval_batch);  in xe_vma_userptr_complete_tlb_inval()
145 	userptr->tlb_inval_submitted = false;  in xe_vma_userptr_complete_tlb_inval()
146 	userptr->finish_inuse = false;  in xe_vma_userptr_complete_tlb_inval()
147 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_complete_tlb_inval()
154 	struct xe_userptr *userptr = &uvma->userptr;  in xe_vma_userptr_invalidate_pass1()  local
168 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vma_userptr_invalidate_pass1()
169 		list_move_tail(&userptr->invalidate_link,  in xe_vma_userptr_invalidate_pass1()
170 			       &vm->userptr.invalidated);  in xe_vma_userptr_invalidate_pass1()
171 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vma_userptr_invalidate_pass1()
194 	if (signaled || userptr->finish_inuse)  in xe_vma_userptr_invalidate_pass1()
198 	userptr->finish_inuse = true;  in xe_vma_userptr_invalidate_pass1()
200 	return &userptr->finish;  in xe_vma_userptr_invalidate_pass1()
208 	struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);  in xe_vma_userptr_invalidate_start()
236 	struct xe_userptr_vma *uvma = container_of(finish, typeof(*uvma), userptr.finish);  in xe_vma_userptr_invalidate_finish()
250 	if (uvma->userptr.tlb_inval_submitted)  in xe_vma_userptr_invalidate_finish()
285 	if (!mmu_interval_read_retry(&uvma->userptr.notifier,  in xe_vma_userptr_force_invalidate()
286 				     uvma->userptr.pages.notifier_seq))  in xe_vma_userptr_force_invalidate()
287 		uvma->userptr.pages.notifier_seq -= 2;  in xe_vma_userptr_force_invalidate()
306 	spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
307 	xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));  in xe_vm_userptr_pin()
308 	list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,  in xe_vm_userptr_pin()
309 				 userptr.invalidate_link) {  in xe_vm_userptr_pin()
310 		list_del_init(&uvma->userptr.invalidate_link);  in xe_vm_userptr_pin()
311 		list_add_tail(&uvma->userptr.repin_link,  in xe_vm_userptr_pin()
312 			      &vm->userptr.repin_list);  in xe_vm_userptr_pin()
314 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
317 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
318 				 userptr.repin_link) {  in xe_vm_userptr_pin()
321 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
351 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
359 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
360 		list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
361 					 userptr.repin_link) {  in xe_vm_userptr_pin()
362 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
363 			list_move_tail(&uvma->userptr.invalidate_link,  in xe_vm_userptr_pin()
364 				       &vm->userptr.invalidated);  in xe_vm_userptr_pin()
366 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
385 	return (list_empty_careful(&vm->userptr.repin_list) &&  in xe_vm_userptr_check_repin()
386 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in xe_vm_userptr_check_repin()
392 	struct xe_userptr *userptr = &uvma->userptr;  in xe_userptr_setup()  local
396 	INIT_LIST_HEAD(&userptr->invalidate_link);  in xe_userptr_setup()
397 	INIT_LIST_HEAD(&userptr->repin_link);  in xe_userptr_setup()
399 	drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm);  in xe_userptr_setup()
401 	err = mmu_interval_notifier_insert(&userptr->notifier, current->mm,  in xe_userptr_setup()
413 	struct xe_userptr *userptr = &uvma->userptr;  in xe_userptr_remove()  local
415 	drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_userptr_remove()
423 	mmu_interval_notifier_remove(&userptr->notifier);  in xe_userptr_remove()
430 	spin_lock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()
431 	xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));  in xe_userptr_destroy()
432 	list_del(&uvma->userptr.invalidate_link);  in xe_userptr_destroy()
433 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()