xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright 2023 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/kref.h>
26 #include <linux/slab.h>
27 #include <linux/dma-fence-unwrap.h>
28 
29 #include <drm/drm_exec.h>
30 #include <drm/drm_syncobj.h>
31 
32 #include "amdgpu.h"
33 #include "amdgpu_trace.h"
34 
35 #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
36 
37 static const struct dma_fence_ops amdgpu_userq_fence_ops;
38 
39 static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
40 {
41 	if (!f || f->ops != &amdgpu_userq_fence_ops)
42 		return NULL;
43 
44 	return container_of(f, struct amdgpu_userq_fence, base);
45 }
46 
47 static u64 amdgpu_userq_fence_read(struct amdgpu_userq_fence_driver *fence_drv)
48 {
49 	return le64_to_cpu(*fence_drv->cpu_addr);
50 }
51 
52 static void
53 amdgpu_userq_fence_write(struct amdgpu_userq_fence_driver *fence_drv,
54 			 u64 seq)
55 {
56 	if (fence_drv->cpu_addr)
57 		*fence_drv->cpu_addr = cpu_to_le64(seq);
58 }
59 
60 int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
61 				    struct amdgpu_userq_fence_driver **fence_drv_req)
62 {
63 	struct amdgpu_userq_fence_driver *fence_drv;
64 	int r;
65 
66 	if (!fence_drv_req)
67 		return -EINVAL;
68 	*fence_drv_req = NULL;
69 
70 	fence_drv = kzalloc_obj(*fence_drv);
71 	if (!fence_drv)
72 		return -ENOMEM;
73 
74 	/* Acquire seq64 memory */
75 	r = amdgpu_seq64_alloc(adev, &fence_drv->va, &fence_drv->gpu_addr,
76 			       &fence_drv->cpu_addr);
77 	if (r)
78 		goto free_fence_drv;
79 
80 	memset(fence_drv->cpu_addr, 0, sizeof(u64));
81 
82 	kref_init(&fence_drv->refcount);
83 	INIT_LIST_HEAD(&fence_drv->fences);
84 	spin_lock_init(&fence_drv->fence_list_lock);
85 
86 	fence_drv->adev = adev;
87 	fence_drv->context = dma_fence_context_alloc(1);
88 	get_task_comm(fence_drv->timeline_name, current);
89 
90 	*fence_drv_req = fence_drv;
91 
92 	return 0;
93 
94 free_fence_drv:
95 	kfree(fence_drv);
96 
97 	return r;
98 }
99 
100 static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa)
101 {
102 	struct amdgpu_userq_fence_driver *fence_drv;
103 	unsigned long index;
104 
105 	if (xa_empty(xa))
106 		return;
107 
108 	xa_lock(xa);
109 	xa_for_each(xa, index, fence_drv) {
110 		__xa_erase(xa, index);
111 		amdgpu_userq_fence_driver_put(fence_drv);
112 	}
113 
114 	xa_unlock(xa);
115 }
116 
117 void
118 amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
119 {
120 	dma_fence_put(userq->last_fence);
121 	userq->last_fence = NULL;
122 	amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
123 	xa_destroy(&userq->fence_drv_xa);
124 	mutex_destroy(&userq->fence_drv_lock);
125 	/* Drop the queue's ownership reference to fence_drv explicitly */
126 	amdgpu_userq_fence_driver_put(userq->fence_drv);
127 }
128 
129 static void
130 amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
131 {
132 	unsigned long i;
133 	for (i = 0; i < userq_fence->fence_drv_array_count; i++)
134 		amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
135 	userq_fence->fence_drv_array_count = 0;
136 }
137 
138 /*
139  * Returns:
140  * -ENOENT when no fences were processes
141  * 1 when more fences are pending
142  * 0 when no fences are pending any more
143  */
144 int
145 amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
146 {
147 	struct amdgpu_userq_fence *userq_fence, *tmp;
148 	LIST_HEAD(to_be_signaled);
149 	struct dma_fence *fence;
150 	unsigned long flags;
151 	u64 fence_val;
152 
153 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
154 	fence_val = amdgpu_userq_fence_read(fence_drv);
155 
156 	list_for_each_entry(userq_fence, &fence_drv->fences, link) {
157 		if (fence_val < userq_fence->base.seqno)
158 			break;
159 	}
160 
161 	list_cut_before(&to_be_signaled, &fence_drv->fences,
162 				&userq_fence->link);
163 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
164 
165 	if (list_empty(&to_be_signaled))
166 		return -ENOENT;
167 
168 	list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
169 		fence = &userq_fence->base;
170 		list_del_init(&userq_fence->link);
171 		dma_fence_signal(fence);
172 		/* Drop fence_drv_array outside fence_list_lock
173 		 * to avoid the recursion lock.
174 		 */
175 		amdgpu_userq_fence_put_fence_drv_array(userq_fence);
176 		dma_fence_put(fence);
177 	}
178 
179 	/* That doesn't need to be accurate so no locking */
180 	return list_empty(&fence_drv->fences) ? 0 : 1;
181 }
182 
183 void amdgpu_userq_fence_driver_destroy(struct kref *ref)
184 {
185 	struct amdgpu_userq_fence_driver *fence_drv = container_of(ref,
186 					 struct amdgpu_userq_fence_driver,
187 					 refcount);
188 	struct amdgpu_device *adev = fence_drv->adev;
189 	struct amdgpu_userq_fence *fence, *tmp;
190 	unsigned long flags;
191 	struct dma_fence *f;
192 
193 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
194 	lockdep_assert_held(&fence_drv->fence_list_lock);
195 	list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) {
196 		f = &fence->base;
197 		spin_lock(dma_fence_spinlock(f));
198 		if (!dma_fence_is_signaled_locked(f)) {
199 			dma_fence_set_error(f, -ECANCELED);
200 			dma_fence_signal_locked(f);
201 		}
202 		spin_unlock(dma_fence_spinlock(f));
203 		list_del(&fence->link);
204 		dma_fence_put(f);
205 	}
206 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
207 
208 	/* Free seq64 memory */
209 	amdgpu_seq64_free(adev, fence_drv->va);
210 	kfree(fence_drv);
211 }
212 
213 void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv)
214 {
215 	kref_get(&fence_drv->refcount);
216 }
217 
218 void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
219 {
220 	kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
221 }
222 
223 static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
224 				    struct amdgpu_userq_fence **pfence)
225 {
226 	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
227 	struct amdgpu_userq_fence *userq_fence;
228 	void *entry;
229 
230 	userq_fence = kmalloc_obj(*userq_fence);
231 	if (!userq_fence)
232 		return -ENOMEM;
233 
234 	/*
235 	 * Get the next unused entry, since we fill from the start this can be
236 	 * used as size to allocate the array.
237 	 */
238 	mutex_lock(&userq->fence_drv_lock);
239 	XA_STATE(xas, &userq->fence_drv_xa, 0);
240 
241 	rcu_read_lock();
242 	do {
243 		entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
244 	} while (xas_retry(&xas, entry));
245 	rcu_read_unlock();
246 
247 	userq_fence->fence_drv_array = kvmalloc_objs(fence_drv, xas.xa_index);
248 	if (!userq_fence->fence_drv_array) {
249 		mutex_unlock(&userq->fence_drv_lock);
250 		kfree(userq_fence);
251 		return -ENOMEM;
252 	}
253 
254 	userq_fence->fence_drv_array_count = xas.xa_index;
255 	xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
256 		   0, ULONG_MAX, xas.xa_index, XA_PRESENT);
257 	xa_destroy(&userq->fence_drv_xa);
258 
259 	mutex_unlock(&userq->fence_drv_lock);
260 
261 	amdgpu_userq_fence_driver_get(fence_drv);
262 	userq_fence->fence_drv = fence_drv;
263 
264 	*pfence = userq_fence;
265 	return 0;
266 }
267 
268 static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
269 				    struct amdgpu_userq_fence *fence,
270 				    u64 seq)
271 {
272 	struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
273 	unsigned long flags;
274 	bool signaled = false;
275 
276 	spin_lock_init(&fence->lock);
277 	dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
278 			 fence_drv->context, seq);
279 
280 	/* Make sure the fence is visible to the hang detect worker */
281 	dma_fence_put(userq->last_fence);
282 	userq->last_fence = dma_fence_get(&fence->base);
283 
284 	/* Check if hardware has already processed the fence */
285 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
286 	if (!dma_fence_is_signaled(&fence->base)) {
287 		dma_fence_get(&fence->base);
288 		list_add_tail(&fence->link, &fence_drv->fences);
289 	} else {
290 		INIT_LIST_HEAD(&fence->link);
291 		signaled = true;
292 	}
293 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
294 
295 	if (signaled)
296 		amdgpu_userq_fence_put_fence_drv_array(fence);
297 	else
298 		amdgpu_userq_start_hang_detect_work(userq);
299 }
300 
301 static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
302 {
303 	return "amdgpu_userq_fence";
304 }
305 
306 static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f)
307 {
308 	struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
309 
310 	return fence->fence_drv->timeline_name;
311 }
312 
313 static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
314 {
315 	struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
316 	struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
317 	u64 fence_val, wptr;
318 
319 	fence_val = amdgpu_userq_fence_read(fence_drv);
320 	wptr = fence->base.seqno;
321 
322 	if (fence_val >= wptr)
323 		return true;
324 
325 	return false;
326 }
327 
328 static void amdgpu_userq_fence_free(struct rcu_head *rcu)
329 {
330 	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
331 	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
332 	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
333 
334 	/* Release the fence driver reference */
335 	amdgpu_userq_fence_driver_put(fence_drv);
336 
337 	kvfree(userq_fence->fence_drv_array);
338 	kfree(userq_fence);
339 }
340 
341 static void amdgpu_userq_fence_release(struct dma_fence *f)
342 {
343 	call_rcu(&f->rcu, amdgpu_userq_fence_free);
344 }
345 
346 static const struct dma_fence_ops amdgpu_userq_fence_ops = {
347 	.get_driver_name = amdgpu_userq_fence_get_driver_name,
348 	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
349 	.signaled = amdgpu_userq_fence_signaled,
350 	.release = amdgpu_userq_fence_release,
351 };
352 
353 /**
354  * amdgpu_userq_fence_read_wptr - Read the userq wptr value
355  *
356  * @adev: amdgpu_device pointer
357  * @queue: user mode queue structure pointer
358  * @wptr: write pointer value
359  *
360  * Read the wptr value from userq's MQD. The userq signal IOCTL
361  * creates a dma_fence for the shared buffers that expects the
362  * fence_val written to seq64 fence address >= WPTR.
363  *
364  * Returns wptr value on success, error on failure.
365  */
366 static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
367 					struct amdgpu_usermode_queue *queue,
368 					u64 *wptr)
369 {
370 	struct amdgpu_bo_va_mapping *mapping;
371 	struct amdgpu_bo *bo;
372 	struct drm_exec exec;
373 	u64 addr, offset, *ptr;
374 	int ret;
375 
376 	addr = queue->userq_prop->wptr_gpu_addr;
377 	addr &= AMDGPU_GMC_HOLE_MASK;
378 
379 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
380 	drm_exec_until_all_locked(&exec) {
381 		ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1);
382 		drm_exec_retry_on_contention(&exec);
383 		if (unlikely(ret))
384 			goto lock_error;
385 
386 		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr);
387 		if (!mapping || !mapping->bo_va->base.bo) {
388 			ret = -EINVAL;
389 			goto lock_error;
390 		}
391 
392 		ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base);
393 		drm_exec_retry_on_contention(&exec);
394 		if (unlikely(ret))
395 			goto lock_error;
396 	}
397 
398 	bo = mapping->bo_va->base.bo;
399 	ret = amdgpu_bo_kmap(bo, (void **)&ptr);
400 	if (ret) {
401 		DRM_ERROR("Failed mapping the userqueue wptr bo");
402 		goto lock_error;
403 	}
404 
405 	/* The WPTR can be anywhere within the BO's VA mapping. */
406 	offset = addr - (mapping->start << AMDGPU_GPU_PAGE_SHIFT);
407 	offset += mapping->offset;
408 	*wptr = le64_to_cpu(ptr[offset / sizeof(*ptr)]);
409 
410 	amdgpu_bo_kunmap(bo);
411 	drm_exec_fini(&exec);
412 	return 0;
413 
414 lock_error:
415 	drm_exec_fini(&exec);
416 	return ret;
417 }
418 
419 static void
420 amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
421 				    int error)
422 {
423 	struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
424 	unsigned long flags;
425 	struct dma_fence *f;
426 
427 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
428 	lockdep_assert_held(&fence_drv->fence_list_lock);
429 	f = rcu_dereference_protected(&fence->base,
430 				      lockdep_is_held(&fence_drv->fence_list_lock));
431 	if (f) {
432 		/* nest f->lock inside fence_list_lock */
433 		spin_lock(dma_fence_spinlock(f));
434 		if (!dma_fence_is_signaled_locked(f))
435 			dma_fence_set_error(f, error);
436 		spin_unlock(dma_fence_spinlock(f));
437 	}
438 	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
439 }
440 
441 void
442 amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq)
443 {
444 	struct dma_fence *f = userq->last_fence;
445 
446 	if (f) {
447 		struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
448 		struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
449 		u64 wptr = fence->base.seqno;
450 
451 		amdgpu_userq_fence_driver_set_error(fence, -ECANCELED);
452 		amdgpu_userq_fence_write(fence_drv, wptr);
453 		amdgpu_userq_fence_driver_process(fence_drv);
454 
455 	}
456 }
457 
458 int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
459 			      struct drm_file *filp)
460 {
461 	struct amdgpu_device *adev = drm_to_adev(dev);
462 	struct drm_amdgpu_userq_signal *args = data;
463 	const unsigned int num_write_bo_handles = args->num_bo_write_handles;
464 	const unsigned int num_read_bo_handles = args->num_bo_read_handles;
465 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
466 	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
467 
468 	struct drm_gem_object **gobj_write, **gobj_read;
469 	u32 *syncobj_handles, num_syncobj_handles;
470 	u64 *syncobj_points;
471 	struct amdgpu_usermode_queue *queue;
472 	struct amdgpu_userq_fence *fence;
473 	struct {
474 		struct drm_syncobj *syncobj;
475 		struct dma_fence_chain *chain;
476 	} *syncobj;
477 	struct drm_exec exec;
478 	void __user *ptr;
479 	int r, i, entry;
480 	u64 wptr;
481 
482 	if (!amdgpu_userq_enabled(dev))
483 		return -ENOTSUPP;
484 
485 	if (args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
486 	    args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
487 		return -EINVAL;
488 
489 	num_syncobj_handles = args->num_syncobj_handles;
490 	ptr = u64_to_user_ptr(args->syncobj_handles);
491 	syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
492 					    sizeof(u32));
493 	if (IS_ERR(syncobj_handles))
494 		return PTR_ERR(syncobj_handles);
495 
496 	if (args->syncobj_points) {
497 		ptr = u64_to_user_ptr(args->syncobj_points);
498 		syncobj_points = memdup_array_user(ptr, num_syncobj_handles,
499 					    sizeof(u64));
500 		if (IS_ERR(syncobj_points)) {
501 			r = PTR_ERR(syncobj_points);
502 			goto free_syncobj_handles;
503 		}
504 	} else {
505 		syncobj_points = NULL;
506 	}
507 
508 	syncobj = kmalloc_objs(*syncobj, num_syncobj_handles);
509 	if (!syncobj) {
510 		r = -ENOMEM;
511 		goto free_syncobj_points;
512 	}
513 
514 	for (entry = 0; entry < num_syncobj_handles; entry++) {
515 		syncobj[entry].chain = NULL;
516 		syncobj[entry].syncobj = drm_syncobj_find(filp, syncobj_handles[entry]);
517 		if (!syncobj[entry].syncobj) {
518 			r = -ENOENT;
519 			goto free_syncobj;
520 		}
521 		if (syncobj_points && syncobj_points[entry]) {
522 			syncobj[entry].chain = dma_fence_chain_alloc();
523 			if (!syncobj[entry].chain) {
524 				drm_syncobj_put(syncobj[entry].syncobj);
525 				r = -ENOMEM;
526 				goto free_syncobj;
527 			}
528 		}
529 	}
530 
531 	ptr = u64_to_user_ptr(args->bo_read_handles);
532 	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
533 	if (r)
534 		goto free_syncobj;
535 
536 	ptr = u64_to_user_ptr(args->bo_write_handles);
537 	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
538 				   &gobj_write);
539 	if (r)
540 		goto put_gobj_read;
541 
542 	queue = amdgpu_userq_get(userq_mgr, args->queue_id);
543 	if (!queue) {
544 		r = -ENOENT;
545 		goto put_gobj_write;
546 	}
547 
548 	r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
549 	if (r)
550 		goto put_queue;
551 
552 	r = amdgpu_userq_fence_alloc(queue, &fence);
553 	if (r)
554 		goto put_queue;
555 
556 	/* We are here means UQ is active, make sure the eviction fence is valid */
557 	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
558 
559 	/* Create the new fence */
560 	amdgpu_userq_fence_init(queue, fence, wptr);
561 
562 	trace_amdgpu_userq_emit_fence(dev->dev, queue, fence);
563 
564 	mutex_unlock(&userq_mgr->userq_mutex);
565 
566 	/*
567 	 * This needs to come after the fence is created since
568 	 * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
569 	 * locks.
570 	 */
571 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
572 		      (num_read_bo_handles + num_write_bo_handles));
573 
574 	drm_exec_until_all_locked(&exec) {
575 		r = drm_exec_prepare_array(&exec, gobj_read,
576 					   num_read_bo_handles, 1);
577 		drm_exec_retry_on_contention(&exec);
578 		if (r)
579 			goto exec_fini;
580 
581 		r = drm_exec_prepare_array(&exec, gobj_write,
582 					   num_write_bo_handles, 1);
583 		drm_exec_retry_on_contention(&exec);
584 		if (r)
585 			goto exec_fini;
586 	}
587 
588 	/* And publish the new fence in the BOs and syncobj */
589 	for (i = 0; i < num_read_bo_handles; i++)
590 		dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
591 				   DMA_RESV_USAGE_READ);
592 
593 	for (i = 0; i < num_write_bo_handles; i++)
594 		dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
595 				   DMA_RESV_USAGE_WRITE);
596 
597 	for (i = 0; i < num_syncobj_handles; i++) {
598 		if (syncobj[i].chain) {
599 			drm_syncobj_add_point(syncobj[i].syncobj, syncobj[i].chain,
600 					   &fence->base, syncobj_points[i]);
601 			syncobj[i].chain = NULL;
602 		} else {
603 			drm_syncobj_replace_fence(syncobj[i].syncobj, &fence->base);
604 		}
605 	}
606 
607 exec_fini:
608 	/* drop the reference acquired in fence creation function */
609 	dma_fence_put(&fence->base);
610 
611 	drm_exec_fini(&exec);
612 put_queue:
613 	amdgpu_userq_put(queue);
614 put_gobj_write:
615 	for (i = 0; i < num_write_bo_handles; i++)
616 		drm_gem_object_put(gobj_write[i]);
617 	kvfree(gobj_write);
618 put_gobj_read:
619 	for (i = 0; i < num_read_bo_handles; i++)
620 		drm_gem_object_put(gobj_read[i]);
621 	kvfree(gobj_read);
622 free_syncobj:
623 	while (entry-- > 0) {
624 		drm_syncobj_put(syncobj[entry].syncobj);
625 		dma_fence_chain_free(syncobj[entry].chain);
626 	}
627 	kfree(syncobj);
628 free_syncobj_points:
629 	kfree(syncobj_points);
630 free_syncobj_handles:
631 	kfree(syncobj_handles);
632 
633 	return r;
634 }
635 
636 /* Count the number of expected fences so userspace can alloc a buffer */
637 static int
638 amdgpu_userq_wait_count_fences(struct drm_file *filp,
639 			       struct drm_amdgpu_userq_wait *wait_info,
640 			       u32 *syncobj_handles, u64 *timeline_points,
641 			       u32 *timeline_handles,
642 			       struct drm_gem_object **gobj_write,
643 			       struct drm_gem_object **gobj_read)
644 {
645 	int num_read_bo_handles, num_write_bo_handles;
646 	struct dma_fence_unwrap iter;
647 	struct dma_fence *fence, *f;
648 	unsigned int num_fences = 0;
649 	struct drm_exec exec;
650 	int i, r;
651 
652 	/*
653 	 * This needs to be outside of the lock provided by drm_exec for
654 	 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
655 	 */
656 
657 	/* Count timeline fences */
658 	for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
659 		r = drm_syncobj_find_fence(filp, timeline_handles[i],
660 					   timeline_points[i],
661 					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
662 					   &fence);
663 		if (r)
664 			return r;
665 
666 		dma_fence_unwrap_for_each(f, &iter, fence)
667 			num_fences++;
668 
669 		dma_fence_put(fence);
670 	}
671 
672 	/* Count boolean fences */
673 	for (i = 0; i < wait_info->num_syncobj_handles; i++) {
674 		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
675 					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
676 					   &fence);
677 		if (r)
678 			return r;
679 
680 		num_fences++;
681 		dma_fence_put(fence);
682 	}
683 
684 	/* Lock all the GEM objects */
685 	/* TODO: It is actually not necessary to lock them */
686 	num_read_bo_handles = wait_info->num_bo_read_handles;
687 	num_write_bo_handles = wait_info->num_bo_write_handles;
688 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
689 		      num_read_bo_handles + num_write_bo_handles);
690 
691 	drm_exec_until_all_locked(&exec) {
692 		r = drm_exec_prepare_array(&exec, gobj_read,
693 					   num_read_bo_handles, 1);
694 		drm_exec_retry_on_contention(&exec);
695 		if (r)
696 			goto error_unlock;
697 
698 		r = drm_exec_prepare_array(&exec, gobj_write,
699 					   num_write_bo_handles, 1);
700 		drm_exec_retry_on_contention(&exec);
701 		if (r)
702 			goto error_unlock;
703 	}
704 
705 	/* Count read fences */
706 	for (i = 0; i < num_read_bo_handles; i++) {
707 		struct dma_resv_iter resv_cursor;
708 		struct dma_fence *fence;
709 
710 		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
711 					DMA_RESV_USAGE_READ, fence)
712 			num_fences++;
713 	}
714 
715 	/* Count write fences */
716 	for (i = 0; i < num_write_bo_handles; i++) {
717 		struct dma_resv_iter resv_cursor;
718 		struct dma_fence *fence;
719 
720 		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
721 					DMA_RESV_USAGE_WRITE, fence)
722 			num_fences++;
723 	}
724 
725 	wait_info->num_fences = min(num_fences, USHRT_MAX);
726 	r = 0;
727 
728 error_unlock:
729 	/* Unlock all the GEM objects */
730 	drm_exec_fini(&exec);
731 	return r;
732 }
733 
734 static int
735 amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info,
736 			    struct dma_fence **fences, unsigned int *num_fences,
737 			    struct dma_fence *fence)
738 {
739 	/* As fallback shouldn't userspace allocate enough space */
740 	if (*num_fences >= wait_info->num_fences)
741 		return dma_fence_wait(fence, true);
742 
743 	fences[(*num_fences)++] = dma_fence_get(fence);
744 	return 0;
745 }
746 
747 static int
748 amdgpu_userq_wait_return_fence_info(struct drm_device *dev, struct drm_file *filp,
749 				    struct drm_amdgpu_userq_wait *wait_info,
750 				    u32 *syncobj_handles, u64 *timeline_points,
751 				    u32 *timeline_handles,
752 				    struct drm_gem_object **gobj_write,
753 				    struct drm_gem_object **gobj_read)
754 {
755 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
756 	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
757 	struct drm_amdgpu_userq_fence_info *fence_info;
758 	int num_read_bo_handles, num_write_bo_handles;
759 	struct amdgpu_usermode_queue *waitq;
760 	struct dma_fence **fences, *fence, *f;
761 	struct dma_fence_unwrap iter;
762 	int num_points, num_syncobj;
763 	unsigned int num_fences = 0;
764 	struct drm_exec exec;
765 	int i, cnt, r;
766 
767 	fence_info = kmalloc_objs(*fence_info, wait_info->num_fences);
768 	if (!fence_info)
769 		return -ENOMEM;
770 
771 	fences = kmalloc_objs(*fences, wait_info->num_fences);
772 	if (!fences) {
773 		r = -ENOMEM;
774 		goto free_fence_info;
775 	}
776 
777 	/* Retrieve timeline fences */
778 	num_points = wait_info->num_syncobj_timeline_handles;
779 	for (i = 0; i < num_points; i++) {
780 		r = drm_syncobj_find_fence(filp, timeline_handles[i],
781 					   timeline_points[i],
782 					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
783 					   &fence);
784 		if (r)
785 			goto free_fences;
786 
787 		dma_fence_unwrap_for_each(f, &iter, fence) {
788 			r = amdgpu_userq_wait_add_fence(wait_info, fences,
789 							&num_fences, f);
790 			if (r) {
791 				dma_fence_put(fence);
792 				goto free_fences;
793 			}
794 		}
795 
796 		dma_fence_put(fence);
797 	}
798 
799 	/* Retrieve boolean fences */
800 	num_syncobj = wait_info->num_syncobj_handles;
801 	for (i = 0; i < num_syncobj; i++) {
802 		struct dma_fence *fence;
803 
804 		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
805 					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
806 					   &fence);
807 		if (r)
808 			goto free_fences;
809 
810 		r = amdgpu_userq_wait_add_fence(wait_info, fences,
811 						&num_fences, fence);
812 		dma_fence_put(fence);
813 		if (r)
814 			goto free_fences;
815 
816 	}
817 
818 	/* Lock all the GEM objects */
819 	num_read_bo_handles = wait_info->num_bo_read_handles;
820 	num_write_bo_handles = wait_info->num_bo_write_handles;
821 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
822 		      num_read_bo_handles + num_write_bo_handles);
823 
824 	drm_exec_until_all_locked(&exec) {
825 		r = drm_exec_prepare_array(&exec, gobj_read,
826 					   num_read_bo_handles, 1);
827 		drm_exec_retry_on_contention(&exec);
828 		if (r)
829 			goto error_unlock;
830 
831 		r = drm_exec_prepare_array(&exec, gobj_write,
832 					   num_write_bo_handles, 1);
833 		drm_exec_retry_on_contention(&exec);
834 		if (r)
835 			goto error_unlock;
836 	}
837 
838 	/* Retrieve GEM read objects fence */
839 	for (i = 0; i < num_read_bo_handles; i++) {
840 		struct dma_resv_iter resv_cursor;
841 		struct dma_fence *fence;
842 
843 		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
844 					DMA_RESV_USAGE_READ, fence) {
845 			r = amdgpu_userq_wait_add_fence(wait_info, fences,
846 							&num_fences, fence);
847 			if (r)
848 				goto error_unlock;
849 		}
850 	}
851 
852 	/* Retrieve GEM write objects fence */
853 	for (i = 0; i < num_write_bo_handles; i++) {
854 		struct dma_resv_iter resv_cursor;
855 		struct dma_fence *fence;
856 
857 		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
858 					DMA_RESV_USAGE_WRITE, fence) {
859 			r = amdgpu_userq_wait_add_fence(wait_info, fences,
860 							&num_fences, fence);
861 			if (r)
862 				goto error_unlock;
863 		}
864 	}
865 
866 	drm_exec_fini(&exec);
867 
868 	/*
869 	 * Keep only the latest fences to reduce the number of values
870 	 * given back to userspace.
871 	 */
872 	num_fences = dma_fence_dedup_array(fences, num_fences);
873 
874 	waitq = amdgpu_userq_get(userq_mgr, wait_info->waitq_id);
875 	if (!waitq) {
876 		r = -EINVAL;
877 		goto free_fences;
878 	}
879 
880 	for (i = 0, cnt = 0; i < num_fences; i++) {
881 		struct amdgpu_userq_fence_driver *fence_drv;
882 		struct amdgpu_userq_fence *userq_fence;
883 		u32 index;
884 
885 		userq_fence = to_amdgpu_userq_fence(fences[i]);
886 		if (!userq_fence) {
887 			/*
888 			 * Just waiting on other driver fences should
889 			 * be good for now
890 			 */
891 			r = dma_fence_wait(fences[i], true);
892 			if (r)
893 				goto put_waitq;
894 
895 			continue;
896 		}
897 
898 		fence_drv = userq_fence->fence_drv;
899 		/*
900 		 * We need to make sure the user queue release their reference
901 		 * to the fence drivers at some point before queue destruction.
902 		 * Otherwise, we would gather those references until we don't
903 		 * have any more space left and crash.
904 		 */
905 		mutex_lock(&waitq->fence_drv_lock);
906 		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
907 			     xa_limit_32b, GFP_KERNEL);
908 		mutex_unlock(&waitq->fence_drv_lock);
909 		if (r)
910 			goto put_waitq;
911 
912 		amdgpu_userq_fence_driver_get(fence_drv);
913 
914 		trace_amdgpu_userq_wait_deps(dev->dev, waitq, userq_fence);
915 
916 		/* Store drm syncobj's gpu va address and value */
917 		fence_info[cnt].va = fence_drv->va;
918 		fence_info[cnt].value = fences[i]->seqno;
919 
920 		/* Increment the actual userq fence count */
921 		cnt++;
922 	}
923 	wait_info->num_fences = cnt;
924 
925 	/* Copy userq fence info to user space */
926 	if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
927 			 fence_info, cnt * sizeof(*fence_info)))
928 		r = -EFAULT;
929 	else
930 		r = 0;
931 
932 put_waitq:
933 	amdgpu_userq_put(waitq);
934 
935 free_fences:
936 	while (num_fences--)
937 		dma_fence_put(fences[num_fences]);
938 	kfree(fences);
939 
940 free_fence_info:
941 	kfree(fence_info);
942 	return r;
943 
944 error_unlock:
945 	drm_exec_fini(&exec);
946 	goto free_fences;
947 }
948 
949 int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
950 			    struct drm_file *filp)
951 {
952 	int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
953 	u32 *syncobj_handles, *timeline_handles;
954 	u64 *timeline_points;
955 	struct drm_amdgpu_userq_wait *wait_info = data;
956 	struct drm_gem_object **gobj_write;
957 	struct drm_gem_object **gobj_read;
958 	void __user *ptr;
959 	int r;
960 
961 	if (!amdgpu_userq_enabled(dev))
962 		return -ENOTSUPP;
963 
964 	if (wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
965 	    wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
966 		return -EINVAL;
967 
968 	num_syncobj = wait_info->num_syncobj_handles;
969 	ptr = u64_to_user_ptr(wait_info->syncobj_handles);
970 	syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
971 	if (IS_ERR(syncobj_handles))
972 		return PTR_ERR(syncobj_handles);
973 
974 	num_points = wait_info->num_syncobj_timeline_handles;
975 	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
976 	timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
977 	if (IS_ERR(timeline_handles)) {
978 		r = PTR_ERR(timeline_handles);
979 		goto free_syncobj_handles;
980 	}
981 
982 	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
983 	timeline_points = memdup_array_user(ptr, num_points, sizeof(u64));
984 	if (IS_ERR(timeline_points)) {
985 		r = PTR_ERR(timeline_points);
986 		goto free_timeline_handles;
987 	}
988 
989 	num_read_bo_handles = wait_info->num_bo_read_handles;
990 	ptr = u64_to_user_ptr(wait_info->bo_read_handles);
991 	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
992 	if (r)
993 		goto free_timeline_points;
994 
995 	num_write_bo_handles = wait_info->num_bo_write_handles;
996 	ptr = u64_to_user_ptr(wait_info->bo_write_handles);
997 	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
998 				   &gobj_write);
999 	if (r)
1000 		goto put_gobj_read;
1001 
1002 	/*
1003 	 * Passing num_fences = 0 means that userspace doesn't want to
1004 	 * retrieve userq_fence_info. If num_fences = 0 we skip filling
1005 	 * userq_fence_info and return the actual number of fences on
1006 	 * args->num_fences.
1007 	 */
1008 	if (!wait_info->num_fences) {
1009 		r = amdgpu_userq_wait_count_fences(filp, wait_info,
1010 						   syncobj_handles,
1011 						   timeline_points,
1012 						   timeline_handles,
1013 						   gobj_write,
1014 						   gobj_read);
1015 	} else {
1016 		r = amdgpu_userq_wait_return_fence_info(dev, filp, wait_info,
1017 							syncobj_handles,
1018 							timeline_points,
1019 							timeline_handles,
1020 							gobj_write,
1021 							gobj_read);
1022 	}
1023 
1024 	while (num_write_bo_handles--)
1025 		drm_gem_object_put(gobj_write[num_write_bo_handles]);
1026 	kvfree(gobj_write);
1027 
1028 put_gobj_read:
1029 	while (num_read_bo_handles--)
1030 		drm_gem_object_put(gobj_read[num_read_bo_handles]);
1031 	kvfree(gobj_read);
1032 
1033 free_timeline_points:
1034 	kfree(timeline_points);
1035 free_timeline_handles:
1036 	kfree(timeline_handles);
1037 free_syncobj_handles:
1038 	kfree(syncobj_handles);
1039 	return r;
1040 }
1041