xref: /linux/drivers/gpu/drm/xe/xe_userptr.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2025 Intel Corporation
4  */
5 
6 #include "xe_svm.h"
7 #include "xe_userptr.h"
8 
9 #include <linux/mm.h>
10 
11 #include "xe_tlb_inval.h"
12 #include "xe_trace_bo.h"
13 
14 static void xe_userptr_assert_in_notifier(struct xe_vm *vm)
15 {
16 	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
17 		       (lockdep_is_held(&vm->lock) &&
18 			lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
19 			dma_resv_held(xe_vm_resv(vm))));
20 }
21 
22 /**
23  * xe_vma_userptr_check_repin() - Advisory check for repin needed
24  * @uvma: The userptr vma
25  *
26  * Check if the userptr vma has been invalidated since last successful
27  * repin. The check is advisory only and can the function can be called
28  * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the
29  * vma userptr will remain valid after a lockless check, so typically
30  * the call needs to be followed by a proper check under the notifier_lock.
31  *
32  * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended.
33  */
34 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma)
35 {
36 	return mmu_interval_check_retry(&uvma->userptr.notifier,
37 					uvma->userptr.pages.notifier_seq) ?
38 		-EAGAIN : 0;
39 }
40 
41 /**
42  * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
43  * that need repinning.
44  * @vm: The VM.
45  *
46  * This function checks for whether the VM has userptrs that need repinning,
47  * and provides a release-type barrier on the svm.gpusvm.notifier_lock after
48  * checking.
49  *
50  * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are.
51  */
52 int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
53 {
54 	lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock);
55 
56 	return (list_empty(&vm->userptr.repin_list) &&
57 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
58 }
59 
60 #if IS_ENABLED(CONFIG_PROVE_LOCKING)
61 static bool __xe_vma_userptr_lockdep(struct xe_userptr_vma *uvma)
62 {
63 	struct xe_vma *vma = &uvma->vma;
64 	struct xe_vm *vm = xe_vma_vm(vma);
65 
66 	return lockdep_is_held_type(&vm->lock, 0) ||
67 		(lockdep_is_held_type(&vm->lock, 1) &&
68 		 lockdep_is_held_type(&vma->fault_lock, 0));
69 }
70 
71 #define xe_vma_userptr_lockdep(uvma)	\
72 	lockdep_assert(__xe_vma_userptr_lockdep(uvma))
73 #else
74 #define xe_vma_userptr_lockdep(uvma)
75 #endif
76 
77 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
78 {
79 	struct xe_vma *vma = &uvma->vma;
80 	struct xe_vm *vm = xe_vma_vm(vma);
81 	struct xe_device *xe = vm->xe;
82 	struct drm_gpusvm_ctx ctx = {
83 		.read_only = xe_vma_read_only(vma),
84 		.device_private_page_owner = xe_svm_private_page_owner(vm, false),
85 		.allow_mixed = true,
86 	};
87 
88 	xe_vma_userptr_lockdep(uvma);
89 	xe_assert(xe, xe_vma_is_userptr(vma));
90 
91 	if (vma->gpuva.flags & XE_VMA_DESTROYED)
92 		return 0;
93 
94 	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 1,
95 				    uvma->userptr.notifier.mm,
96 				    &uvma->userptr.notifier,
97 				    xe_vma_userptr(vma),
98 				    xe_vma_userptr(vma) + xe_vma_size(vma),
99 				    &ctx);
100 }
101 
102 static struct mmu_interval_notifier_finish *
103 xe_vma_userptr_do_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma, bool is_deferred)
104 {
105 	struct xe_userptr *userptr = &uvma->userptr;
106 	struct xe_vma *vma = &uvma->vma;
107 	struct drm_gpusvm_ctx ctx = {
108 		.in_notifier = true,
109 		.read_only = xe_vma_read_only(vma),
110 	};
111 	long err;
112 
113 	xe_userptr_assert_in_notifier(vm);
114 	if (is_deferred)
115 		xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted);
116 
117 	err = dma_resv_wait_timeout(xe_vm_resv(vm),
118 				    DMA_RESV_USAGE_BOOKKEEP,
119 				    false, MAX_SCHEDULE_TIMEOUT);
120 	XE_WARN_ON(err <= 0);
121 
122 	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
123 		if (!userptr->finish_inuse) {
124 			/*
125 			 * Defer the TLB wait to an extra pass so the caller
126 			 * can pipeline TLB flushes across GPUs before waiting
127 			 * on any of them.
128 			 */
129 			xe_assert(vm->xe, !userptr->tlb_inval_submitted);
130 			userptr->finish_inuse = true;
131 			userptr->tlb_inval_submitted = true;
132 			err = xe_vm_invalidate_vma_submit(vma, &userptr->inval_batch);
133 			XE_WARN_ON(err);
134 			return &userptr->finish;
135 		}
136 		err = xe_vm_invalidate_vma(vma);
137 		XE_WARN_ON(err);
138 	}
139 
140 	if (is_deferred)
141 		userptr->finish_inuse = false;
142 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
143 			       xe_vma_size(vma) >> PAGE_SHIFT, &ctx);
144 	return NULL;
145 }
146 
147 static void
148 xe_vma_userptr_complete_tlb_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma)
149 {
150 	struct xe_userptr *userptr = &uvma->userptr;
151 	struct xe_vma *vma = &uvma->vma;
152 	struct drm_gpusvm_ctx ctx = {
153 		.in_notifier = true,
154 		.read_only = xe_vma_read_only(vma),
155 	};
156 
157 	xe_userptr_assert_in_notifier(vm);
158 	xe_assert(vm->xe, userptr->finish_inuse);
159 	xe_assert(vm->xe, userptr->tlb_inval_submitted);
160 
161 	xe_tlb_inval_batch_wait(&userptr->inval_batch);
162 	userptr->tlb_inval_submitted = false;
163 	userptr->finish_inuse = false;
164 	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
165 			       xe_vma_size(vma) >> PAGE_SHIFT, &ctx);
166 }
167 
168 static struct mmu_interval_notifier_finish *
169 xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma)
170 {
171 	struct xe_userptr *userptr = &uvma->userptr;
172 	struct xe_vma *vma = &uvma->vma;
173 	struct dma_resv_iter cursor;
174 	struct dma_fence *fence;
175 	bool signaled = true;
176 
177 	xe_userptr_assert_in_notifier(vm);
178 
179 	/*
180 	 * Tell exec and rebind worker they need to repin and rebind this
181 	 * userptr.
182 	 */
183 	if (!xe_vm_in_fault_mode(vm) &&
184 	    !(vma->gpuva.flags & XE_VMA_DESTROYED)) {
185 		spin_lock(&vm->userptr.invalidated_lock);
186 		list_move_tail(&userptr->invalidate_link,
187 			       &vm->userptr.invalidated);
188 		spin_unlock(&vm->userptr.invalidated_lock);
189 	}
190 
191 	/*
192 	 * Preempt fences turn into schedule disables, pipeline these.
193 	 * Note that even in fault mode, we need to wait for binds and
194 	 * unbinds to complete, and those are attached as BOOKMARK fences
195 	 * to the vm.
196 	 */
197 	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
198 			    DMA_RESV_USAGE_BOOKKEEP);
199 	dma_resv_for_each_fence_unlocked(&cursor, fence) {
200 		dma_fence_enable_signaling(fence);
201 		if (signaled && !dma_fence_is_signaled(fence))
202 			signaled = false;
203 	}
204 	dma_resv_iter_end(&cursor);
205 
206 	/*
207 	 * Only one caller at a time can use the multi-pass state.
208 	 * If it's already in use, or all fences are already signaled,
209 	 * proceed directly to invalidation without deferring.
210 	 */
211 	if (signaled || userptr->finish_inuse)
212 		return xe_vma_userptr_do_inval(vm, uvma, false);
213 
214 	/* Defer: the notifier core will call invalidate_finish once done. */
215 	userptr->finish_inuse = true;
216 
217 	return &userptr->finish;
218 }
219 
220 static bool xe_vma_userptr_invalidate_start(struct mmu_interval_notifier *mni,
221 					    const struct mmu_notifier_range *range,
222 					    unsigned long cur_seq,
223 					    struct mmu_interval_notifier_finish **p_finish)
224 {
225 	struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);
226 	struct xe_vma *vma = &uvma->vma;
227 	struct xe_vm *vm = xe_vma_vm(vma);
228 
229 	xe_assert(vm->xe, xe_vma_is_userptr(vma));
230 	trace_xe_vma_userptr_invalidate(vma);
231 
232 	if (!mmu_notifier_range_blockable(range))
233 		return false;
234 
235 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
236 	       "NOTIFIER PASS1: addr=0x%016llx, range=0x%016llx",
237 		xe_vma_start(vma), xe_vma_size(vma));
238 
239 	down_write(&vm->svm.gpusvm.notifier_lock);
240 	mmu_interval_set_seq(mni, cur_seq);
241 
242 	*p_finish = xe_vma_userptr_invalidate_pass1(vm, uvma);
243 
244 	up_write(&vm->svm.gpusvm.notifier_lock);
245 	if (!*p_finish)
246 		trace_xe_vma_userptr_invalidate_complete(vma);
247 
248 	return true;
249 }
250 
251 static void xe_vma_userptr_invalidate_finish(struct mmu_interval_notifier_finish *finish)
252 {
253 	struct xe_userptr_vma *uvma = container_of(finish, typeof(*uvma), userptr.finish);
254 	struct xe_vma *vma = &uvma->vma;
255 	struct xe_vm *vm = xe_vma_vm(vma);
256 
257 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
258 	       "NOTIFIER PASS2: addr=0x%016llx, range=0x%016llx",
259 		xe_vma_start(vma), xe_vma_size(vma));
260 
261 	down_write(&vm->svm.gpusvm.notifier_lock);
262 	/*
263 	 * If a TLB invalidation was previously submitted (deferred from the
264 	 * synchronous pass1 fallback), wait for it and unmap pages.
265 	 * Otherwise, fences have now completed: invalidate the TLB and unmap.
266 	 */
267 	if (uvma->userptr.tlb_inval_submitted)
268 		xe_vma_userptr_complete_tlb_inval(vm, uvma);
269 	else
270 		xe_vma_userptr_do_inval(vm, uvma, true);
271 	up_write(&vm->svm.gpusvm.notifier_lock);
272 	trace_xe_vma_userptr_invalidate_complete(vma);
273 }
274 
275 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
276 	.invalidate_start = xe_vma_userptr_invalidate_start,
277 	.invalidate_finish = xe_vma_userptr_invalidate_finish,
278 };
279 
280 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
281 /**
282  * xe_vma_userptr_force_invalidate() - force invalidate a userptr
283  * @uvma: The userptr vma to invalidate
284  *
285  * Perform a forced userptr invalidation for testing purposes.
286  */
287 void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
288 {
289 	struct mmu_interval_notifier_finish *finish;
290 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
291 
292 	/* Protect against concurrent userptr pinning */
293 	lockdep_assert_held(&vm->lock);
294 	/* Protect against concurrent notifiers */
295 	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
296 	/*
297 	 * Protect against concurrent instances of this function and
298 	 * the critical exec sections
299 	 */
300 	xe_vm_assert_held(vm);
301 
302 	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
303 				     uvma->userptr.pages.notifier_seq))
304 		uvma->userptr.pages.notifier_seq -= 2;
305 
306 	finish = xe_vma_userptr_invalidate_pass1(vm, uvma);
307 	if (finish)
308 		finish = xe_vma_userptr_do_inval(vm, uvma, true);
309 	if (finish)
310 		xe_vma_userptr_complete_tlb_inval(vm, uvma);
311 }
312 #endif
313 
314 int xe_vm_userptr_pin(struct xe_vm *vm)
315 {
316 	struct xe_userptr_vma *uvma, *next;
317 	int err = 0;
318 
319 	xe_assert(vm->xe, !xe_vm_in_fault_mode(vm));
320 	lockdep_assert_held_write(&vm->lock);
321 
322 	/* Collect invalidated userptrs */
323 	spin_lock(&vm->userptr.invalidated_lock);
324 	xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));
325 	list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,
326 				 userptr.invalidate_link) {
327 		list_del_init(&uvma->userptr.invalidate_link);
328 		list_add_tail(&uvma->userptr.repin_link,
329 			      &vm->userptr.repin_list);
330 	}
331 	spin_unlock(&vm->userptr.invalidated_lock);
332 
333 	/* Pin and move to bind list */
334 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,
335 				 userptr.repin_link) {
336 		err = xe_vma_userptr_pin_pages(uvma);
337 		if (err == -EFAULT) {
338 			list_del_init(&uvma->userptr.repin_link);
339 			/*
340 			 * We might have already done the pin once already, but
341 			 * then had to retry before the re-bind happened, due
342 			 * some other condition in the caller, but in the
343 			 * meantime the userptr got dinged by the notifier such
344 			 * that we need to revalidate here, but this time we hit
345 			 * the EFAULT. In such a case make sure we remove
346 			 * ourselves from the rebind list to avoid going down in
347 			 * flames.
348 			 */
349 			if (!list_empty(&uvma->vma.combined_links.rebind))
350 				list_del_init(&uvma->vma.combined_links.rebind);
351 
352 			/* Wait for pending binds */
353 			xe_vm_lock(vm, false);
354 			dma_resv_wait_timeout(xe_vm_resv(vm),
355 					      DMA_RESV_USAGE_BOOKKEEP,
356 					      false, MAX_SCHEDULE_TIMEOUT);
357 
358 			down_read(&vm->svm.gpusvm.notifier_lock);
359 			err = xe_vm_invalidate_vma(&uvma->vma);
360 			up_read(&vm->svm.gpusvm.notifier_lock);
361 			xe_vm_unlock(vm);
362 			if (err)
363 				break;
364 		} else {
365 			if (err)
366 				break;
367 
368 			list_del_init(&uvma->userptr.repin_link);
369 			list_move_tail(&uvma->vma.combined_links.rebind,
370 				       &vm->rebind_list);
371 		}
372 	}
373 
374 	if (err) {
375 		down_write(&vm->svm.gpusvm.notifier_lock);
376 		spin_lock(&vm->userptr.invalidated_lock);
377 		list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,
378 					 userptr.repin_link) {
379 			list_del_init(&uvma->userptr.repin_link);
380 			list_move_tail(&uvma->userptr.invalidate_link,
381 				       &vm->userptr.invalidated);
382 		}
383 		spin_unlock(&vm->userptr.invalidated_lock);
384 		up_write(&vm->svm.gpusvm.notifier_lock);
385 	}
386 	return err;
387 }
388 
389 /**
390  * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
391  * that need repinning.
392  * @vm: The VM.
393  *
394  * This function does an advisory check for whether the VM has userptrs that
395  * need repinning.
396  *
397  * Return: 0 if there are no indications of userptrs needing repinning,
398  * -EAGAIN if there are.
399  */
400 int xe_vm_userptr_check_repin(struct xe_vm *vm)
401 {
402 	return (list_empty_careful(&vm->userptr.repin_list) &&
403 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
404 }
405 
406 int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start,
407 		     unsigned long range)
408 {
409 	struct xe_userptr *userptr = &uvma->userptr;
410 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
411 	int err;
412 
413 	INIT_LIST_HEAD(&userptr->invalidate_link);
414 	INIT_LIST_HEAD(&userptr->repin_link);
415 
416 	drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm);
417 
418 	err = mmu_interval_notifier_insert(&userptr->notifier, current->mm,
419 					   start, range,
420 					   &vma_userptr_notifier_ops);
421 	if (err)
422 		return err;
423 
424 	return 0;
425 }
426 
427 void xe_userptr_remove(struct xe_userptr_vma *uvma)
428 {
429 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
430 	struct xe_userptr *userptr = &uvma->userptr;
431 
432 	drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
433 			      xe_vma_size(&uvma->vma) >> PAGE_SHIFT);
434 
435 	/*
436 	 * Since userptr pages are not pinned, we can't remove
437 	 * the notifier until we're sure the GPU is not accessing
438 	 * them anymore
439 	 */
440 	mmu_interval_notifier_remove(&userptr->notifier);
441 }
442 
443 void xe_userptr_destroy(struct xe_userptr_vma *uvma)
444 {
445 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
446 
447 	spin_lock(&vm->userptr.invalidated_lock);
448 	xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));
449 	list_del(&uvma->userptr.invalidate_link);
450 	spin_unlock(&vm->userptr.invalidated_lock);
451 }
452