1 // SPDX-License-Identifier: MIT
13  * xe_vma_userptr_check_repin() - Advisory check for repin needed
18  * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the
22  * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended.
26 	return mmu_interval_check_retry(&uvma->userptr.notifier,  in xe_vma_userptr_check_repin()
27 					uvma->userptr.pages.notifier_seq) ?  in xe_vma_userptr_check_repin()
28 		-EAGAIN : 0;  in xe_vma_userptr_check_repin()
32  * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
37  * and provides a release-type barrier on the svm.gpusvm.notifier_lock after
40  * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are.
44 	lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock);  in __xe_vm_userptr_needs_repin()
46 	return (list_empty(&vm->userptr.repin_list) &&  in __xe_vm_userptr_needs_repin()
47 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in __xe_vm_userptr_needs_repin()
52 	struct xe_vma *vma = &uvma->vma;  in xe_vma_userptr_pin_pages()
54 	struct xe_device *xe = vm->xe;  in xe_vma_userptr_pin_pages()
60 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_pin_pages()
63 	if (vma->gpuva.flags & XE_VMA_DESTROYED)  in xe_vma_userptr_pin_pages()
66 	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_vma_userptr_pin_pages()
67 				    uvma->userptr.notifier.mm,  in xe_vma_userptr_pin_pages()
68 				    &uvma->userptr.notifier,  in xe_vma_userptr_pin_pages()
76 	struct xe_userptr *userptr = &uvma->userptr;  in __vma_userptr_invalidate()
77 	struct xe_vma *vma = &uvma->vma;  in __vma_userptr_invalidate()
87 	 * Tell exec and rebind worker they need to repin and rebind this  in __vma_userptr_invalidate()
91 	    !(vma->gpuva.flags & XE_VMA_DESTROYED)) {  in __vma_userptr_invalidate()
92 		spin_lock(&vm->userptr.invalidated_lock);  in __vma_userptr_invalidate()
93 		list_move_tail(&userptr->invalidate_link,  in __vma_userptr_invalidate()
94 			       &vm->userptr.invalidated);  in __vma_userptr_invalidate()
95 		spin_unlock(&vm->userptr.invalidated_lock);  in __vma_userptr_invalidate()
115 	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {  in __vma_userptr_invalidate()
120 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in __vma_userptr_invalidate()
129 	struct xe_vma *vma = &uvma->vma;  in vma_userptr_invalidate()
132 	xe_assert(vm->xe, xe_vma_is_userptr(vma));  in vma_userptr_invalidate()
138 	vm_dbg(&xe_vma_vm(vma)->xe->drm,  in vma_userptr_invalidate()
142 	down_write(&vm->svm.gpusvm.notifier_lock);  in vma_userptr_invalidate()
146 	up_write(&vm->svm.gpusvm.notifier_lock);  in vma_userptr_invalidate()
158  * xe_vma_userptr_force_invalidate() - force invalidate a userptr
165 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_vma_userptr_force_invalidate()
167 	/* Protect against concurrent userptr pinning */  in xe_vma_userptr_force_invalidate()
168 	lockdep_assert_held(&vm->lock);  in xe_vma_userptr_force_invalidate()
169 	/* Protect against concurrent notifiers */  in xe_vma_userptr_force_invalidate()
170 	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);  in xe_vma_userptr_force_invalidate()
172 	 * Protect against concurrent instances of this function and  in xe_vma_userptr_force_invalidate()
173 	 * the critical exec sections  in xe_vma_userptr_force_invalidate()
177 	if (!mmu_interval_read_retry(&uvma->userptr.notifier,  in xe_vma_userptr_force_invalidate()
178 				     uvma->userptr.pages.notifier_seq))  in xe_vma_userptr_force_invalidate()
179 		uvma->userptr.pages.notifier_seq -= 2;  in xe_vma_userptr_force_invalidate()
189 	xe_assert(vm->xe, !xe_vm_in_fault_mode(vm));  in xe_vm_userptr_pin()
190 	lockdep_assert_held_write(&vm->lock);  in xe_vm_userptr_pin()
193 	spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
194 	xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));  in xe_vm_userptr_pin()
195 	list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,  in xe_vm_userptr_pin()
197 		list_del_init(&uvma->userptr.invalidate_link);  in xe_vm_userptr_pin()
198 		list_add_tail(&uvma->userptr.repin_link,  in xe_vm_userptr_pin()
199 			      &vm->userptr.repin_list);  in xe_vm_userptr_pin()
201 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
204 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
207 		if (err == -EFAULT) {  in xe_vm_userptr_pin()
208 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
211 			 * then had to retry before the re-bind happened, due  in xe_vm_userptr_pin()
219 			if (!list_empty(&uvma->vma.combined_links.rebind))  in xe_vm_userptr_pin()
220 				list_del_init(&uvma->vma.combined_links.rebind);  in xe_vm_userptr_pin()
228 			down_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
229 			err = xe_vm_invalidate_vma(&uvma->vma);  in xe_vm_userptr_pin()
230 			up_read(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
238 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
239 			list_move_tail(&uvma->vma.combined_links.rebind,  in xe_vm_userptr_pin()
240 				       &vm->rebind_list);  in xe_vm_userptr_pin()
245 		down_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
246 		spin_lock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
247 		list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,  in xe_vm_userptr_pin()
249 			list_del_init(&uvma->userptr.repin_link);  in xe_vm_userptr_pin()
250 			list_move_tail(&uvma->userptr.invalidate_link,  in xe_vm_userptr_pin()
251 				       &vm->userptr.invalidated);  in xe_vm_userptr_pin()
253 		spin_unlock(&vm->userptr.invalidated_lock);  in xe_vm_userptr_pin()
254 		up_write(&vm->svm.gpusvm.notifier_lock);  in xe_vm_userptr_pin()
260  * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
268  * -EAGAIN if there are.
272 	return (list_empty_careful(&vm->userptr.repin_list) &&  in xe_vm_userptr_check_repin()
273 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;  in xe_vm_userptr_check_repin()
279 	struct xe_userptr *userptr = &uvma->userptr;  in xe_userptr_setup()
282 	INIT_LIST_HEAD(&userptr->invalidate_link);  in xe_userptr_setup()
283 	INIT_LIST_HEAD(&userptr->repin_link);  in xe_userptr_setup()
285 	err = mmu_interval_notifier_insert(&userptr->notifier, current->mm,  in xe_userptr_setup()
291 	userptr->pages.notifier_seq = LONG_MAX;  in xe_userptr_setup()
298 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_remove()
299 	struct xe_userptr *userptr = &uvma->userptr;  in xe_userptr_remove()
301 	drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages,  in xe_userptr_remove()
302 			      xe_vma_size(&uvma->vma) >> PAGE_SHIFT);  in xe_userptr_remove()
309 	mmu_interval_notifier_remove(&userptr->notifier);  in xe_userptr_remove()
314 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);  in xe_userptr_destroy()
316 	spin_lock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()
317 	xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));  in xe_userptr_destroy()
318 	list_del(&uvma->userptr.invalidate_link);  in xe_userptr_destroy()
319 	spin_unlock(&vm->userptr.invalidated_lock);  in xe_userptr_destroy()