xref: /linux/drivers/gpu/drm/drm_syncobj.c (revision 42b16d3ac371a2fac9b6f08fd75f23f34ba3955a)
1 /*
2  * Copyright 2017 Red Hat
3  * Parts ported from amdgpu (fence wait code).
4  * Copyright 2016 Advanced Micro Devices, Inc.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23  * IN THE SOFTWARE.
24  *
25  * Authors:
26  *
27  */
28 
29 /**
30  * DOC: Overview
31  *
32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
33  * container for a synchronization primitive which can be used by userspace
34  * to explicitly synchronize GPU commands, can be shared between userspace
35  * processes, and can be shared between different DRM drivers.
36  * Their primary use-case is to implement Vulkan fences and semaphores.
37  * The syncobj userspace API provides ioctls for several operations:
38  *
39  *  - Creation and destruction of syncobjs
40  *  - Import and export of syncobjs to/from a syncobj file descriptor
41  *  - Import and export a syncobj's underlying fence to/from a sync file
42  *  - Reset a syncobj (set its fence to NULL)
43  *  - Signal a syncobj (set a trivially signaled fence)
44  *  - Wait for a syncobj's fence to appear and be signaled
45  *
46  * The syncobj userspace API also provides operations to manipulate a syncobj
47  * in terms of a timeline of struct &dma_fence_chain rather than a single
48  * struct &dma_fence, through the following operations:
49  *
50  *   - Signal a given point on the timeline
51  *   - Wait for a given point to appear and/or be signaled
52  *   - Import and export from/to a given point of a timeline
53  *
54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
55  * &dma_fence which may be NULL.
56  * When a syncobj is first created, its pointer is either NULL or a pointer
57  * to an already signaled fence depending on whether the
58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
59  * &DRM_IOCTL_SYNCOBJ_CREATE.
60  *
61  * If the syncobj is considered as a binary (its state is either signaled or
62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
63  * the syncobj, the syncobj's fence is replaced with a fence which will be
64  * signaled by the completion of that work.
65  * If the syncobj is considered as a timeline primitive, when GPU work is
66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
68  * pointing to the previous fence that was in the syncobj. The new struct
69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
70  * completion of the DRM driver's work and also any work associated with the
71  * fence previously in the syncobj.
72  *
73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
74  * time the work is enqueued, it waits on the syncobj's fence before
75  * submitting the work to hardware. That fence is either :
76  *
77  *    - The syncobj's current fence if the syncobj is considered as a binary
78  *      primitive.
79  *    - The struct &dma_fence associated with a given point if the syncobj is
80  *      considered as a timeline primitive.
81  *
82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
83  * the enqueue operation is expected to fail.
84  *
85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
86  * terms of the current fence at the time the ioctl is called by userspace
87  * regardless of whether that operation is an immediate host-side operation
88  * (signal or reset) or or an operation which is enqueued in some driver
89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
91  * setting its pointer to a fence which is already signaled.
92  *
93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
94  * terms of a u64 value referring to point in the timeline. See
95  * dma_fence_chain_find_seqno() to see how a given point is found in the
96  * timeline.
97  *
98  * Note that applications should be careful to always use timeline set of
99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
100  * set of ioctl() with a syncobj considered as timeline could result incorrect
101  * synchronization. The use of binary syncobj is supported through the
102  * timeline set of ioctl() by using a point value of 0, this will reproduce
103  * the behavior of the binary set of ioctl() (for example replace the
104  * syncobj's fence when signaling).
105  *
106  *
107  * Host-side wait on syncobjs
108  * --------------------------
109  *
110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
111  * host-side wait on all of the syncobj fences simultaneously.
112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
113  * all of the syncobj fences to be signaled before it returns.
114  * Otherwise, it returns once at least one syncobj fence has been signaled
115  * and the index of a signaled fence is written back to the client.
116  *
117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
119  * the host-side wait will first wait for the syncobj to receive a non-NULL
120  * fence and then wait on that fence.
121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
123  * Assuming the syncobj starts off with a NULL fence, this allows a client
124  * to do a host wait in one thread (or process) which waits on GPU work
125  * submitted in another thread (or process) without having to manually
126  * synchronize between the two.
127  * This requirement is inherited from the Vulkan fence API.
128  *
129  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set
130  * a fence deadline hint on the backing fences before waiting, to provide the
131  * fence signaler with an appropriate sense of urgency.  The deadline is
132  * specified as an absolute &CLOCK_MONOTONIC value in units of ns.
133  *
134  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
135  * handles as well as an array of u64 points and does a host-side wait on all
136  * of syncobj fences at the given points simultaneously.
137  *
138  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
139  * fence to materialize on the timeline without waiting for the fence to be
140  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
141  * requirement is inherited from the wait-before-signal behavior required by
142  * the Vulkan timeline semaphore API.
143  *
144  * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
145  * blocking: an eventfd will be signaled when the syncobj is. This is useful to
146  * integrate the wait in an event loop.
147  *
148  *
149  * Import/export of syncobjs
150  * -------------------------
151  *
152  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
153  * provide two mechanisms for import/export of syncobjs.
154  *
155  * The first lets the client import or export an entire syncobj to a file
156  * descriptor.
157  * These fd's are opaque and have no other use case, except passing the
158  * syncobj between processes.
159  * All exported file descriptors and any syncobj handles created as a
160  * result of importing those file descriptors own a reference to the
161  * same underlying struct &drm_syncobj and the syncobj can be used
162  * persistently across all the processes with which it is shared.
163  * The syncobj is freed only once the last reference is dropped.
164  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
165  * reference) for every import instead of de-duplicating.
166  * The primary use-case of this persistent import/export is for shared
167  * Vulkan fences and semaphores.
168  *
169  * The second import/export mechanism, which is indicated by
170  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
171  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
172  * import/export the syncobj's current fence from/to a &sync_file.
173  * When a syncobj is exported to a sync file, that sync file wraps the
174  * sycnobj's fence at the time of export and any later signal or reset
175  * operations on the syncobj will not affect the exported sync file.
176  * When a sync file is imported into a syncobj, the syncobj's fence is set
177  * to the fence wrapped by that sync file.
178  * Because sync files are immutable, resetting or signaling the syncobj
179  * will not affect any sync files whose fences have been imported into the
180  * syncobj.
181  *
182  *
183  * Import/export of timeline points in timeline syncobjs
184  * -----------------------------------------------------
185  *
186  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
187  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
188  * into another syncobj.
189  *
190  * Note that if you want to transfer a struct &dma_fence_chain from a given
191  * point on a timeline syncobj from/into a binary syncobj, you can use the
192  * point 0 to mean take/replace the fence in the syncobj.
193  */
194 
195 #include <linux/anon_inodes.h>
196 #include <linux/dma-fence-unwrap.h>
197 #include <linux/eventfd.h>
198 #include <linux/file.h>
199 #include <linux/fs.h>
200 #include <linux/sched/signal.h>
201 #include <linux/sync_file.h>
202 #include <linux/uaccess.h>
203 
204 #include <drm/drm.h>
205 #include <drm/drm_drv.h>
206 #include <drm/drm_file.h>
207 #include <drm/drm_gem.h>
208 #include <drm/drm_print.h>
209 #include <drm/drm_syncobj.h>
210 #include <drm/drm_utils.h>
211 
212 #include "drm_internal.h"
213 
214 struct syncobj_wait_entry {
215 	struct list_head node;
216 	struct task_struct *task;
217 	struct dma_fence *fence;
218 	struct dma_fence_cb fence_cb;
219 	u64    point;
220 };
221 
222 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
223 				      struct syncobj_wait_entry *wait);
224 
225 struct syncobj_eventfd_entry {
226 	struct list_head node;
227 	struct dma_fence *fence;
228 	struct dma_fence_cb fence_cb;
229 	struct drm_syncobj *syncobj;
230 	struct eventfd_ctx *ev_fd_ctx;
231 	u64 point;
232 	u32 flags;
233 };
234 
235 static void
236 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
237 			   struct syncobj_eventfd_entry *entry);
238 
239 /**
240  * drm_syncobj_find - lookup and reference a sync object.
241  * @file_private: drm file private pointer
242  * @handle: sync object handle to lookup.
243  *
244  * Returns a reference to the syncobj pointed to by handle or NULL. The
245  * reference must be released by calling drm_syncobj_put().
246  */
drm_syncobj_find(struct drm_file * file_private,u32 handle)247 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
248 				     u32 handle)
249 {
250 	struct drm_syncobj *syncobj;
251 
252 	spin_lock(&file_private->syncobj_table_lock);
253 
254 	/* Check if we currently have a reference on the object */
255 	syncobj = idr_find(&file_private->syncobj_idr, handle);
256 	if (syncobj)
257 		drm_syncobj_get(syncobj);
258 
259 	spin_unlock(&file_private->syncobj_table_lock);
260 
261 	return syncobj;
262 }
263 EXPORT_SYMBOL(drm_syncobj_find);
264 
drm_syncobj_fence_add_wait(struct drm_syncobj * syncobj,struct syncobj_wait_entry * wait)265 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
266 				       struct syncobj_wait_entry *wait)
267 {
268 	struct dma_fence *fence;
269 
270 	if (wait->fence)
271 		return;
272 
273 	spin_lock(&syncobj->lock);
274 	/* We've already tried once to get a fence and failed.  Now that we
275 	 * have the lock, try one more time just to be sure we don't add a
276 	 * callback when a fence has already been set.
277 	 */
278 	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
279 	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
280 		dma_fence_put(fence);
281 		list_add_tail(&wait->node, &syncobj->cb_list);
282 	} else if (!fence) {
283 		wait->fence = dma_fence_get_stub();
284 	} else {
285 		wait->fence = fence;
286 	}
287 	spin_unlock(&syncobj->lock);
288 }
289 
drm_syncobj_remove_wait(struct drm_syncobj * syncobj,struct syncobj_wait_entry * wait)290 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
291 				    struct syncobj_wait_entry *wait)
292 {
293 	if (!wait->node.next)
294 		return;
295 
296 	spin_lock(&syncobj->lock);
297 	list_del_init(&wait->node);
298 	spin_unlock(&syncobj->lock);
299 }
300 
301 static void
syncobj_eventfd_entry_free(struct syncobj_eventfd_entry * entry)302 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
303 {
304 	eventfd_ctx_put(entry->ev_fd_ctx);
305 	dma_fence_put(entry->fence);
306 	/* This happens either inside the syncobj lock, or after the node has
307 	 * already been removed from the list.
308 	 */
309 	list_del(&entry->node);
310 	kfree(entry);
311 }
312 
313 static void
drm_syncobj_add_eventfd(struct drm_syncobj * syncobj,struct syncobj_eventfd_entry * entry)314 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
315 			struct syncobj_eventfd_entry *entry)
316 {
317 	spin_lock(&syncobj->lock);
318 	list_add_tail(&entry->node, &syncobj->ev_fd_list);
319 	syncobj_eventfd_entry_func(syncobj, entry);
320 	spin_unlock(&syncobj->lock);
321 }
322 
323 /**
324  * drm_syncobj_add_point - add new timeline point to the syncobj
325  * @syncobj: sync object to add timeline point do
326  * @chain: chain node to use to add the point
327  * @fence: fence to encapsulate in the chain node
328  * @point: sequence number to use for the point
329  *
330  * Add the chain node as new timeline point to the syncobj.
331  */
drm_syncobj_add_point(struct drm_syncobj * syncobj,struct dma_fence_chain * chain,struct dma_fence * fence,uint64_t point)332 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
333 			   struct dma_fence_chain *chain,
334 			   struct dma_fence *fence,
335 			   uint64_t point)
336 {
337 	struct syncobj_wait_entry *wait_cur, *wait_tmp;
338 	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
339 	struct dma_fence *prev;
340 
341 	dma_fence_get(fence);
342 
343 	spin_lock(&syncobj->lock);
344 
345 	prev = drm_syncobj_fence_get(syncobj);
346 	/* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
347 	if (prev && prev->seqno >= point)
348 		DRM_DEBUG("You are adding an unorder point to timeline!\n");
349 	dma_fence_chain_init(chain, prev, fence, point);
350 	rcu_assign_pointer(syncobj->fence, &chain->base);
351 
352 	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
353 		syncobj_wait_syncobj_func(syncobj, wait_cur);
354 	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
355 		syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
356 	spin_unlock(&syncobj->lock);
357 
358 	/* Walk the chain once to trigger garbage collection */
359 	dma_fence_chain_for_each(fence, prev);
360 	dma_fence_put(prev);
361 }
362 EXPORT_SYMBOL(drm_syncobj_add_point);
363 
364 /**
365  * drm_syncobj_replace_fence - replace fence in a sync object.
366  * @syncobj: Sync object to replace fence in
367  * @fence: fence to install in sync file.
368  *
369  * This replaces the fence on a sync object.
370  */
drm_syncobj_replace_fence(struct drm_syncobj * syncobj,struct dma_fence * fence)371 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
372 			       struct dma_fence *fence)
373 {
374 	struct dma_fence *old_fence;
375 	struct syncobj_wait_entry *wait_cur, *wait_tmp;
376 	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
377 
378 	if (fence)
379 		dma_fence_get(fence);
380 
381 	spin_lock(&syncobj->lock);
382 
383 	old_fence = rcu_dereference_protected(syncobj->fence,
384 					      lockdep_is_held(&syncobj->lock));
385 	rcu_assign_pointer(syncobj->fence, fence);
386 
387 	if (fence != old_fence) {
388 		list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
389 			syncobj_wait_syncobj_func(syncobj, wait_cur);
390 		list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
391 			syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
392 	}
393 
394 	spin_unlock(&syncobj->lock);
395 
396 	dma_fence_put(old_fence);
397 }
398 EXPORT_SYMBOL(drm_syncobj_replace_fence);
399 
400 /**
401  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
402  * @syncobj: sync object to assign the fence on
403  *
404  * Assign a already signaled stub fence to the sync object.
405  */
drm_syncobj_assign_null_handle(struct drm_syncobj * syncobj)406 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
407 {
408 	struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
409 
410 	if (!fence)
411 		return -ENOMEM;
412 
413 	drm_syncobj_replace_fence(syncobj, fence);
414 	dma_fence_put(fence);
415 	return 0;
416 }
417 
418 /* 5s default for wait submission */
419 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
420 /**
421  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
422  * @file_private: drm file private pointer
423  * @handle: sync object handle to lookup.
424  * @point: timeline point
425  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
426  * @fence: out parameter for the fence
427  *
428  * This is just a convenience function that combines drm_syncobj_find() and
429  * drm_syncobj_fence_get().
430  *
431  * Returns 0 on success or a negative error value on failure. On success @fence
432  * contains a reference to the fence, which must be released by calling
433  * dma_fence_put().
434  */
drm_syncobj_find_fence(struct drm_file * file_private,u32 handle,u64 point,u64 flags,struct dma_fence ** fence)435 int drm_syncobj_find_fence(struct drm_file *file_private,
436 			   u32 handle, u64 point, u64 flags,
437 			   struct dma_fence **fence)
438 {
439 	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
440 	struct syncobj_wait_entry wait;
441 	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
442 	int ret;
443 
444 	if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
445 		return -EINVAL;
446 
447 	if (!syncobj)
448 		return -ENOENT;
449 
450 	/* Waiting for userspace with locks help is illegal cause that can
451 	 * trivial deadlock with page faults for example. Make lockdep complain
452 	 * about it early on.
453 	 */
454 	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
455 		might_sleep();
456 		lockdep_assert_none_held_once();
457 	}
458 
459 	*fence = drm_syncobj_fence_get(syncobj);
460 
461 	if (*fence) {
462 		ret = dma_fence_chain_find_seqno(fence, point);
463 		if (!ret) {
464 			/* If the requested seqno is already signaled
465 			 * drm_syncobj_find_fence may return a NULL
466 			 * fence. To make sure the recipient gets
467 			 * signalled, use a new fence instead.
468 			 */
469 			if (!*fence)
470 				*fence = dma_fence_get_stub();
471 
472 			goto out;
473 		}
474 		dma_fence_put(*fence);
475 	} else {
476 		ret = -EINVAL;
477 	}
478 
479 	if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
480 		goto out;
481 
482 	memset(&wait, 0, sizeof(wait));
483 	wait.task = current;
484 	wait.point = point;
485 	drm_syncobj_fence_add_wait(syncobj, &wait);
486 
487 	do {
488 		set_current_state(TASK_INTERRUPTIBLE);
489 		if (wait.fence) {
490 			ret = 0;
491 			break;
492 		}
493                 if (timeout == 0) {
494                         ret = -ETIME;
495                         break;
496                 }
497 
498 		if (signal_pending(current)) {
499 			ret = -ERESTARTSYS;
500 			break;
501 		}
502 
503                 timeout = schedule_timeout(timeout);
504 	} while (1);
505 
506 	__set_current_state(TASK_RUNNING);
507 	*fence = wait.fence;
508 
509 	if (wait.node.next)
510 		drm_syncobj_remove_wait(syncobj, &wait);
511 
512 out:
513 	drm_syncobj_put(syncobj);
514 
515 	return ret;
516 }
517 EXPORT_SYMBOL(drm_syncobj_find_fence);
518 
519 /**
520  * drm_syncobj_free - free a sync object.
521  * @kref: kref to free.
522  *
523  * Only to be called from kref_put in drm_syncobj_put.
524  */
drm_syncobj_free(struct kref * kref)525 void drm_syncobj_free(struct kref *kref)
526 {
527 	struct drm_syncobj *syncobj = container_of(kref,
528 						   struct drm_syncobj,
529 						   refcount);
530 	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
531 
532 	drm_syncobj_replace_fence(syncobj, NULL);
533 
534 	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
535 		syncobj_eventfd_entry_free(ev_fd_cur);
536 
537 	kfree(syncobj);
538 }
539 EXPORT_SYMBOL(drm_syncobj_free);
540 
541 /**
542  * drm_syncobj_create - create a new syncobj
543  * @out_syncobj: returned syncobj
544  * @flags: DRM_SYNCOBJ_* flags
545  * @fence: if non-NULL, the syncobj will represent this fence
546  *
547  * This is the first function to create a sync object. After creating, drivers
548  * probably want to make it available to userspace, either through
549  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
550  *
551  * Returns 0 on success or a negative error value on failure.
552  */
drm_syncobj_create(struct drm_syncobj ** out_syncobj,uint32_t flags,struct dma_fence * fence)553 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
554 		       struct dma_fence *fence)
555 {
556 	int ret;
557 	struct drm_syncobj *syncobj;
558 
559 	syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
560 	if (!syncobj)
561 		return -ENOMEM;
562 
563 	kref_init(&syncobj->refcount);
564 	INIT_LIST_HEAD(&syncobj->cb_list);
565 	INIT_LIST_HEAD(&syncobj->ev_fd_list);
566 	spin_lock_init(&syncobj->lock);
567 
568 	if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
569 		ret = drm_syncobj_assign_null_handle(syncobj);
570 		if (ret < 0) {
571 			drm_syncobj_put(syncobj);
572 			return ret;
573 		}
574 	}
575 
576 	if (fence)
577 		drm_syncobj_replace_fence(syncobj, fence);
578 
579 	*out_syncobj = syncobj;
580 	return 0;
581 }
582 EXPORT_SYMBOL(drm_syncobj_create);
583 
584 /**
585  * drm_syncobj_get_handle - get a handle from a syncobj
586  * @file_private: drm file private pointer
587  * @syncobj: Sync object to export
588  * @handle: out parameter with the new handle
589  *
590  * Exports a sync object created with drm_syncobj_create() as a handle on
591  * @file_private to userspace.
592  *
593  * Returns 0 on success or a negative error value on failure.
594  */
drm_syncobj_get_handle(struct drm_file * file_private,struct drm_syncobj * syncobj,u32 * handle)595 int drm_syncobj_get_handle(struct drm_file *file_private,
596 			   struct drm_syncobj *syncobj, u32 *handle)
597 {
598 	int ret;
599 
600 	/* take a reference to put in the idr */
601 	drm_syncobj_get(syncobj);
602 
603 	idr_preload(GFP_KERNEL);
604 	spin_lock(&file_private->syncobj_table_lock);
605 	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
606 	spin_unlock(&file_private->syncobj_table_lock);
607 
608 	idr_preload_end();
609 
610 	if (ret < 0) {
611 		drm_syncobj_put(syncobj);
612 		return ret;
613 	}
614 
615 	*handle = ret;
616 	return 0;
617 }
618 EXPORT_SYMBOL(drm_syncobj_get_handle);
619 
drm_syncobj_create_as_handle(struct drm_file * file_private,u32 * handle,uint32_t flags)620 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
621 					u32 *handle, uint32_t flags)
622 {
623 	int ret;
624 	struct drm_syncobj *syncobj;
625 
626 	ret = drm_syncobj_create(&syncobj, flags, NULL);
627 	if (ret)
628 		return ret;
629 
630 	ret = drm_syncobj_get_handle(file_private, syncobj, handle);
631 	drm_syncobj_put(syncobj);
632 	return ret;
633 }
634 
drm_syncobj_destroy(struct drm_file * file_private,u32 handle)635 static int drm_syncobj_destroy(struct drm_file *file_private,
636 			       u32 handle)
637 {
638 	struct drm_syncobj *syncobj;
639 
640 	spin_lock(&file_private->syncobj_table_lock);
641 	syncobj = idr_remove(&file_private->syncobj_idr, handle);
642 	spin_unlock(&file_private->syncobj_table_lock);
643 
644 	if (!syncobj)
645 		return -EINVAL;
646 
647 	drm_syncobj_put(syncobj);
648 	return 0;
649 }
650 
drm_syncobj_file_release(struct inode * inode,struct file * file)651 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
652 {
653 	struct drm_syncobj *syncobj = file->private_data;
654 
655 	drm_syncobj_put(syncobj);
656 	return 0;
657 }
658 
659 static const struct file_operations drm_syncobj_file_fops = {
660 	.release = drm_syncobj_file_release,
661 };
662 
663 /**
664  * drm_syncobj_get_fd - get a file descriptor from a syncobj
665  * @syncobj: Sync object to export
666  * @p_fd: out parameter with the new file descriptor
667  *
668  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
669  *
670  * Returns 0 on success or a negative error value on failure.
671  */
drm_syncobj_get_fd(struct drm_syncobj * syncobj,int * p_fd)672 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
673 {
674 	struct file *file;
675 	int fd;
676 
677 	fd = get_unused_fd_flags(O_CLOEXEC);
678 	if (fd < 0)
679 		return fd;
680 
681 	file = anon_inode_getfile("syncobj_file",
682 				  &drm_syncobj_file_fops,
683 				  syncobj, 0);
684 	if (IS_ERR(file)) {
685 		put_unused_fd(fd);
686 		return PTR_ERR(file);
687 	}
688 
689 	drm_syncobj_get(syncobj);
690 	fd_install(fd, file);
691 
692 	*p_fd = fd;
693 	return 0;
694 }
695 EXPORT_SYMBOL(drm_syncobj_get_fd);
696 
drm_syncobj_handle_to_fd(struct drm_file * file_private,u32 handle,int * p_fd)697 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
698 				    u32 handle, int *p_fd)
699 {
700 	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
701 	int ret;
702 
703 	if (!syncobj)
704 		return -EINVAL;
705 
706 	ret = drm_syncobj_get_fd(syncobj, p_fd);
707 	drm_syncobj_put(syncobj);
708 	return ret;
709 }
710 
drm_syncobj_fd_to_handle(struct drm_file * file_private,int fd,u32 * handle)711 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
712 				    int fd, u32 *handle)
713 {
714 	struct drm_syncobj *syncobj;
715 	struct fd f = fdget(fd);
716 	int ret;
717 
718 	if (!fd_file(f))
719 		return -EINVAL;
720 
721 	if (fd_file(f)->f_op != &drm_syncobj_file_fops) {
722 		fdput(f);
723 		return -EINVAL;
724 	}
725 
726 	/* take a reference to put in the idr */
727 	syncobj = fd_file(f)->private_data;
728 	drm_syncobj_get(syncobj);
729 
730 	idr_preload(GFP_KERNEL);
731 	spin_lock(&file_private->syncobj_table_lock);
732 	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
733 	spin_unlock(&file_private->syncobj_table_lock);
734 	idr_preload_end();
735 
736 	if (ret > 0) {
737 		*handle = ret;
738 		ret = 0;
739 	} else
740 		drm_syncobj_put(syncobj);
741 
742 	fdput(f);
743 	return ret;
744 }
745 
drm_syncobj_import_sync_file_fence(struct drm_file * file_private,int fd,int handle)746 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
747 					      int fd, int handle)
748 {
749 	struct dma_fence *fence = sync_file_get_fence(fd);
750 	struct drm_syncobj *syncobj;
751 
752 	if (!fence)
753 		return -EINVAL;
754 
755 	syncobj = drm_syncobj_find(file_private, handle);
756 	if (!syncobj) {
757 		dma_fence_put(fence);
758 		return -ENOENT;
759 	}
760 
761 	drm_syncobj_replace_fence(syncobj, fence);
762 	dma_fence_put(fence);
763 	drm_syncobj_put(syncobj);
764 	return 0;
765 }
766 
drm_syncobj_export_sync_file(struct drm_file * file_private,int handle,int * p_fd)767 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
768 					int handle, int *p_fd)
769 {
770 	int ret;
771 	struct dma_fence *fence;
772 	struct sync_file *sync_file;
773 	int fd = get_unused_fd_flags(O_CLOEXEC);
774 
775 	if (fd < 0)
776 		return fd;
777 
778 	ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
779 	if (ret)
780 		goto err_put_fd;
781 
782 	sync_file = sync_file_create(fence);
783 
784 	dma_fence_put(fence);
785 
786 	if (!sync_file) {
787 		ret = -EINVAL;
788 		goto err_put_fd;
789 	}
790 
791 	fd_install(fd, sync_file->file);
792 
793 	*p_fd = fd;
794 	return 0;
795 err_put_fd:
796 	put_unused_fd(fd);
797 	return ret;
798 }
799 /**
800  * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
801  * @file_private: drm file-private structure to set up
802  *
803  * Called at device open time, sets up the structure for handling refcounting
804  * of sync objects.
805  */
806 void
drm_syncobj_open(struct drm_file * file_private)807 drm_syncobj_open(struct drm_file *file_private)
808 {
809 	idr_init_base(&file_private->syncobj_idr, 1);
810 	spin_lock_init(&file_private->syncobj_table_lock);
811 }
812 
813 static int
drm_syncobj_release_handle(int id,void * ptr,void * data)814 drm_syncobj_release_handle(int id, void *ptr, void *data)
815 {
816 	struct drm_syncobj *syncobj = ptr;
817 
818 	drm_syncobj_put(syncobj);
819 	return 0;
820 }
821 
822 /**
823  * drm_syncobj_release - release file-private sync object resources
824  * @file_private: drm file-private structure to clean up
825  *
826  * Called at close time when the filp is going away.
827  *
828  * Releases any remaining references on objects by this filp.
829  */
830 void
drm_syncobj_release(struct drm_file * file_private)831 drm_syncobj_release(struct drm_file *file_private)
832 {
833 	idr_for_each(&file_private->syncobj_idr,
834 		     &drm_syncobj_release_handle, file_private);
835 	idr_destroy(&file_private->syncobj_idr);
836 }
837 
838 int
drm_syncobj_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)839 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
840 			 struct drm_file *file_private)
841 {
842 	struct drm_syncobj_create *args = data;
843 
844 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
845 		return -EOPNOTSUPP;
846 
847 	/* no valid flags yet */
848 	if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
849 		return -EINVAL;
850 
851 	return drm_syncobj_create_as_handle(file_private,
852 					    &args->handle, args->flags);
853 }
854 
855 int
drm_syncobj_destroy_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)856 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
857 			  struct drm_file *file_private)
858 {
859 	struct drm_syncobj_destroy *args = data;
860 
861 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
862 		return -EOPNOTSUPP;
863 
864 	/* make sure padding is empty */
865 	if (args->pad)
866 		return -EINVAL;
867 	return drm_syncobj_destroy(file_private, args->handle);
868 }
869 
870 int
drm_syncobj_handle_to_fd_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)871 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
872 				   struct drm_file *file_private)
873 {
874 	struct drm_syncobj_handle *args = data;
875 
876 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
877 		return -EOPNOTSUPP;
878 
879 	if (args->pad)
880 		return -EINVAL;
881 
882 	if (args->flags != 0 &&
883 	    args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
884 		return -EINVAL;
885 
886 	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
887 		return drm_syncobj_export_sync_file(file_private, args->handle,
888 						    &args->fd);
889 
890 	return drm_syncobj_handle_to_fd(file_private, args->handle,
891 					&args->fd);
892 }
893 
894 int
drm_syncobj_fd_to_handle_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)895 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
896 				   struct drm_file *file_private)
897 {
898 	struct drm_syncobj_handle *args = data;
899 
900 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
901 		return -EOPNOTSUPP;
902 
903 	if (args->pad)
904 		return -EINVAL;
905 
906 	if (args->flags != 0 &&
907 	    args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
908 		return -EINVAL;
909 
910 	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
911 		return drm_syncobj_import_sync_file_fence(file_private,
912 							  args->fd,
913 							  args->handle);
914 
915 	return drm_syncobj_fd_to_handle(file_private, args->fd,
916 					&args->handle);
917 }
918 
drm_syncobj_transfer_to_timeline(struct drm_file * file_private,struct drm_syncobj_transfer * args)919 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
920 					    struct drm_syncobj_transfer *args)
921 {
922 	struct drm_syncobj *timeline_syncobj = NULL;
923 	struct dma_fence *fence, *tmp;
924 	struct dma_fence_chain *chain;
925 	int ret;
926 
927 	timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
928 	if (!timeline_syncobj) {
929 		return -ENOENT;
930 	}
931 	ret = drm_syncobj_find_fence(file_private, args->src_handle,
932 				     args->src_point, args->flags,
933 				     &tmp);
934 	if (ret)
935 		goto err_put_timeline;
936 
937 	fence = dma_fence_unwrap_merge(tmp);
938 	dma_fence_put(tmp);
939 	if (!fence) {
940 		ret = -ENOMEM;
941 		goto err_put_timeline;
942 	}
943 
944 	chain = dma_fence_chain_alloc();
945 	if (!chain) {
946 		ret = -ENOMEM;
947 		goto err_free_fence;
948 	}
949 
950 	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
951 err_free_fence:
952 	dma_fence_put(fence);
953 err_put_timeline:
954 	drm_syncobj_put(timeline_syncobj);
955 
956 	return ret;
957 }
958 
959 static int
drm_syncobj_transfer_to_binary(struct drm_file * file_private,struct drm_syncobj_transfer * args)960 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
961 			       struct drm_syncobj_transfer *args)
962 {
963 	struct drm_syncobj *binary_syncobj = NULL;
964 	struct dma_fence *fence;
965 	int ret;
966 
967 	binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
968 	if (!binary_syncobj)
969 		return -ENOENT;
970 	ret = drm_syncobj_find_fence(file_private, args->src_handle,
971 				     args->src_point, args->flags, &fence);
972 	if (ret)
973 		goto err;
974 	drm_syncobj_replace_fence(binary_syncobj, fence);
975 	dma_fence_put(fence);
976 err:
977 	drm_syncobj_put(binary_syncobj);
978 
979 	return ret;
980 }
981 int
drm_syncobj_transfer_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)982 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
983 			   struct drm_file *file_private)
984 {
985 	struct drm_syncobj_transfer *args = data;
986 	int ret;
987 
988 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
989 		return -EOPNOTSUPP;
990 
991 	if (args->pad)
992 		return -EINVAL;
993 
994 	if (args->dst_point)
995 		ret = drm_syncobj_transfer_to_timeline(file_private, args);
996 	else
997 		ret = drm_syncobj_transfer_to_binary(file_private, args);
998 
999 	return ret;
1000 }
1001 
syncobj_wait_fence_func(struct dma_fence * fence,struct dma_fence_cb * cb)1002 static void syncobj_wait_fence_func(struct dma_fence *fence,
1003 				    struct dma_fence_cb *cb)
1004 {
1005 	struct syncobj_wait_entry *wait =
1006 		container_of(cb, struct syncobj_wait_entry, fence_cb);
1007 
1008 	wake_up_process(wait->task);
1009 }
1010 
syncobj_wait_syncobj_func(struct drm_syncobj * syncobj,struct syncobj_wait_entry * wait)1011 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1012 				      struct syncobj_wait_entry *wait)
1013 {
1014 	struct dma_fence *fence;
1015 
1016 	/* This happens inside the syncobj lock */
1017 	fence = rcu_dereference_protected(syncobj->fence,
1018 					  lockdep_is_held(&syncobj->lock));
1019 	dma_fence_get(fence);
1020 	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1021 		dma_fence_put(fence);
1022 		return;
1023 	} else if (!fence) {
1024 		wait->fence = dma_fence_get_stub();
1025 	} else {
1026 		wait->fence = fence;
1027 	}
1028 
1029 	wake_up_process(wait->task);
1030 	list_del_init(&wait->node);
1031 }
1032 
drm_syncobj_array_wait_timeout(struct drm_syncobj ** syncobjs,void __user * user_points,uint32_t count,uint32_t flags,signed long timeout,uint32_t * idx,ktime_t * deadline)1033 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1034 						  void __user *user_points,
1035 						  uint32_t count,
1036 						  uint32_t flags,
1037 						  signed long timeout,
1038 						  uint32_t *idx,
1039 						  ktime_t *deadline)
1040 {
1041 	struct syncobj_wait_entry *entries;
1042 	struct dma_fence *fence;
1043 	uint64_t *points;
1044 	uint32_t signaled_count, i;
1045 
1046 	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1047 		     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1048 		might_sleep();
1049 		lockdep_assert_none_held_once();
1050 	}
1051 
1052 	points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1053 	if (points == NULL)
1054 		return -ENOMEM;
1055 
1056 	if (!user_points) {
1057 		memset(points, 0, count * sizeof(uint64_t));
1058 
1059 	} else if (copy_from_user(points, user_points,
1060 				  sizeof(uint64_t) * count)) {
1061 		timeout = -EFAULT;
1062 		goto err_free_points;
1063 	}
1064 
1065 	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1066 	if (!entries) {
1067 		timeout = -ENOMEM;
1068 		goto err_free_points;
1069 	}
1070 	/* Walk the list of sync objects and initialize entries.  We do
1071 	 * this up-front so that we can properly return -EINVAL if there is
1072 	 * a syncobj with a missing fence and then never have the chance of
1073 	 * returning -EINVAL again.
1074 	 */
1075 	signaled_count = 0;
1076 	for (i = 0; i < count; ++i) {
1077 		struct dma_fence *fence;
1078 
1079 		entries[i].task = current;
1080 		entries[i].point = points[i];
1081 		fence = drm_syncobj_fence_get(syncobjs[i]);
1082 		if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1083 			dma_fence_put(fence);
1084 			if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1085 				     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1086 				continue;
1087 			} else {
1088 				timeout = -EINVAL;
1089 				goto cleanup_entries;
1090 			}
1091 		}
1092 
1093 		if (fence)
1094 			entries[i].fence = fence;
1095 		else
1096 			entries[i].fence = dma_fence_get_stub();
1097 
1098 		if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1099 		    dma_fence_is_signaled(entries[i].fence)) {
1100 			if (signaled_count == 0 && idx)
1101 				*idx = i;
1102 			signaled_count++;
1103 		}
1104 	}
1105 
1106 	if (signaled_count == count ||
1107 	    (signaled_count > 0 &&
1108 	     !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1109 		goto cleanup_entries;
1110 
1111 	/* There's a very annoying laxness in the dma_fence API here, in
1112 	 * that backends are not required to automatically report when a
1113 	 * fence is signaled prior to fence->ops->enable_signaling() being
1114 	 * called.  So here if we fail to match signaled_count, we need to
1115 	 * fallthough and try a 0 timeout wait!
1116 	 */
1117 
1118 	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1119 		     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1120 		for (i = 0; i < count; ++i)
1121 			drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1122 	}
1123 
1124 	if (deadline) {
1125 		for (i = 0; i < count; ++i) {
1126 			fence = entries[i].fence;
1127 			if (!fence)
1128 				continue;
1129 			dma_fence_set_deadline(fence, *deadline);
1130 		}
1131 	}
1132 
1133 	do {
1134 		set_current_state(TASK_INTERRUPTIBLE);
1135 
1136 		signaled_count = 0;
1137 		for (i = 0; i < count; ++i) {
1138 			fence = entries[i].fence;
1139 			if (!fence)
1140 				continue;
1141 
1142 			if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1143 			    dma_fence_is_signaled(fence) ||
1144 			    (!entries[i].fence_cb.func &&
1145 			     dma_fence_add_callback(fence,
1146 						    &entries[i].fence_cb,
1147 						    syncobj_wait_fence_func))) {
1148 				/* The fence has been signaled */
1149 				if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1150 					signaled_count++;
1151 				} else {
1152 					if (idx)
1153 						*idx = i;
1154 					goto done_waiting;
1155 				}
1156 			}
1157 		}
1158 
1159 		if (signaled_count == count)
1160 			goto done_waiting;
1161 
1162 		if (timeout == 0) {
1163 			timeout = -ETIME;
1164 			goto done_waiting;
1165 		}
1166 
1167 		if (signal_pending(current)) {
1168 			timeout = -ERESTARTSYS;
1169 			goto done_waiting;
1170 		}
1171 
1172 		timeout = schedule_timeout(timeout);
1173 	} while (1);
1174 
1175 done_waiting:
1176 	__set_current_state(TASK_RUNNING);
1177 
1178 cleanup_entries:
1179 	for (i = 0; i < count; ++i) {
1180 		drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1181 		if (entries[i].fence_cb.func)
1182 			dma_fence_remove_callback(entries[i].fence,
1183 						  &entries[i].fence_cb);
1184 		dma_fence_put(entries[i].fence);
1185 	}
1186 	kfree(entries);
1187 
1188 err_free_points:
1189 	kfree(points);
1190 
1191 	return timeout;
1192 }
1193 
1194 /**
1195  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1196  *
1197  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1198  *
1199  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1200  */
drm_timeout_abs_to_jiffies(int64_t timeout_nsec)1201 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1202 {
1203 	ktime_t abs_timeout, now;
1204 	u64 timeout_ns, timeout_jiffies64;
1205 
1206 	/* make 0 timeout means poll - absolute 0 doesn't seem valid */
1207 	if (timeout_nsec == 0)
1208 		return 0;
1209 
1210 	abs_timeout = ns_to_ktime(timeout_nsec);
1211 	now = ktime_get();
1212 
1213 	if (!ktime_after(abs_timeout, now))
1214 		return 0;
1215 
1216 	timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1217 
1218 	timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1219 	/*  clamp timeout to avoid infinite timeout */
1220 	if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1221 		return MAX_SCHEDULE_TIMEOUT - 1;
1222 
1223 	return timeout_jiffies64 + 1;
1224 }
1225 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1226 
drm_syncobj_array_wait(struct drm_device * dev,struct drm_file * file_private,struct drm_syncobj_wait * wait,struct drm_syncobj_timeline_wait * timeline_wait,struct drm_syncobj ** syncobjs,bool timeline,ktime_t * deadline)1227 static int drm_syncobj_array_wait(struct drm_device *dev,
1228 				  struct drm_file *file_private,
1229 				  struct drm_syncobj_wait *wait,
1230 				  struct drm_syncobj_timeline_wait *timeline_wait,
1231 				  struct drm_syncobj **syncobjs, bool timeline,
1232 				  ktime_t *deadline)
1233 {
1234 	signed long timeout = 0;
1235 	uint32_t first = ~0;
1236 
1237 	if (!timeline) {
1238 		timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1239 		timeout = drm_syncobj_array_wait_timeout(syncobjs,
1240 							 NULL,
1241 							 wait->count_handles,
1242 							 wait->flags,
1243 							 timeout, &first,
1244 							 deadline);
1245 		if (timeout < 0)
1246 			return timeout;
1247 		wait->first_signaled = first;
1248 	} else {
1249 		timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1250 		timeout = drm_syncobj_array_wait_timeout(syncobjs,
1251 							 u64_to_user_ptr(timeline_wait->points),
1252 							 timeline_wait->count_handles,
1253 							 timeline_wait->flags,
1254 							 timeout, &first,
1255 							 deadline);
1256 		if (timeout < 0)
1257 			return timeout;
1258 		timeline_wait->first_signaled = first;
1259 	}
1260 	return 0;
1261 }
1262 
drm_syncobj_array_find(struct drm_file * file_private,void __user * user_handles,uint32_t count_handles,struct drm_syncobj *** syncobjs_out)1263 static int drm_syncobj_array_find(struct drm_file *file_private,
1264 				  void __user *user_handles,
1265 				  uint32_t count_handles,
1266 				  struct drm_syncobj ***syncobjs_out)
1267 {
1268 	uint32_t i, *handles;
1269 	struct drm_syncobj **syncobjs;
1270 	int ret;
1271 
1272 	handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1273 	if (handles == NULL)
1274 		return -ENOMEM;
1275 
1276 	if (copy_from_user(handles, user_handles,
1277 			   sizeof(uint32_t) * count_handles)) {
1278 		ret = -EFAULT;
1279 		goto err_free_handles;
1280 	}
1281 
1282 	syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1283 	if (syncobjs == NULL) {
1284 		ret = -ENOMEM;
1285 		goto err_free_handles;
1286 	}
1287 
1288 	for (i = 0; i < count_handles; i++) {
1289 		syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1290 		if (!syncobjs[i]) {
1291 			ret = -ENOENT;
1292 			goto err_put_syncobjs;
1293 		}
1294 	}
1295 
1296 	kfree(handles);
1297 	*syncobjs_out = syncobjs;
1298 	return 0;
1299 
1300 err_put_syncobjs:
1301 	while (i-- > 0)
1302 		drm_syncobj_put(syncobjs[i]);
1303 	kfree(syncobjs);
1304 err_free_handles:
1305 	kfree(handles);
1306 
1307 	return ret;
1308 }
1309 
drm_syncobj_array_free(struct drm_syncobj ** syncobjs,uint32_t count)1310 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1311 				   uint32_t count)
1312 {
1313 	uint32_t i;
1314 
1315 	for (i = 0; i < count; i++)
1316 		drm_syncobj_put(syncobjs[i]);
1317 	kfree(syncobjs);
1318 }
1319 
1320 int
drm_syncobj_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1321 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1322 		       struct drm_file *file_private)
1323 {
1324 	struct drm_syncobj_wait *args = data;
1325 	struct drm_syncobj **syncobjs;
1326 	unsigned int possible_flags;
1327 	ktime_t t, *tp = NULL;
1328 	int ret = 0;
1329 
1330 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1331 		return -EOPNOTSUPP;
1332 
1333 	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1334 			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1335 			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1336 
1337 	if (args->flags & ~possible_flags)
1338 		return -EINVAL;
1339 
1340 	if (args->count_handles == 0)
1341 		return 0;
1342 
1343 	ret = drm_syncobj_array_find(file_private,
1344 				     u64_to_user_ptr(args->handles),
1345 				     args->count_handles,
1346 				     &syncobjs);
1347 	if (ret < 0)
1348 		return ret;
1349 
1350 	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1351 		t = ns_to_ktime(args->deadline_nsec);
1352 		tp = &t;
1353 	}
1354 
1355 	ret = drm_syncobj_array_wait(dev, file_private,
1356 				     args, NULL, syncobjs, false, tp);
1357 
1358 	drm_syncobj_array_free(syncobjs, args->count_handles);
1359 
1360 	return ret;
1361 }
1362 
1363 int
drm_syncobj_timeline_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1364 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1365 				struct drm_file *file_private)
1366 {
1367 	struct drm_syncobj_timeline_wait *args = data;
1368 	struct drm_syncobj **syncobjs;
1369 	unsigned int possible_flags;
1370 	ktime_t t, *tp = NULL;
1371 	int ret = 0;
1372 
1373 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1374 		return -EOPNOTSUPP;
1375 
1376 	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1377 			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1378 			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE |
1379 			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1380 
1381 	if (args->flags & ~possible_flags)
1382 		return -EINVAL;
1383 
1384 	if (args->count_handles == 0)
1385 		return 0;
1386 
1387 	ret = drm_syncobj_array_find(file_private,
1388 				     u64_to_user_ptr(args->handles),
1389 				     args->count_handles,
1390 				     &syncobjs);
1391 	if (ret < 0)
1392 		return ret;
1393 
1394 	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1395 		t = ns_to_ktime(args->deadline_nsec);
1396 		tp = &t;
1397 	}
1398 
1399 	ret = drm_syncobj_array_wait(dev, file_private,
1400 				     NULL, args, syncobjs, true, tp);
1401 
1402 	drm_syncobj_array_free(syncobjs, args->count_handles);
1403 
1404 	return ret;
1405 }
1406 
syncobj_eventfd_entry_fence_func(struct dma_fence * fence,struct dma_fence_cb * cb)1407 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1408 					     struct dma_fence_cb *cb)
1409 {
1410 	struct syncobj_eventfd_entry *entry =
1411 		container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1412 
1413 	eventfd_signal(entry->ev_fd_ctx);
1414 	syncobj_eventfd_entry_free(entry);
1415 }
1416 
1417 static void
syncobj_eventfd_entry_func(struct drm_syncobj * syncobj,struct syncobj_eventfd_entry * entry)1418 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1419 			   struct syncobj_eventfd_entry *entry)
1420 {
1421 	int ret;
1422 	struct dma_fence *fence;
1423 
1424 	/* This happens inside the syncobj lock */
1425 	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1426 	if (!fence)
1427 		return;
1428 
1429 	ret = dma_fence_chain_find_seqno(&fence, entry->point);
1430 	if (ret != 0) {
1431 		/* The given seqno has not been submitted yet. */
1432 		dma_fence_put(fence);
1433 		return;
1434 	} else if (!fence) {
1435 		/* If dma_fence_chain_find_seqno returns 0 but sets the fence
1436 		 * to NULL, it implies that the given seqno is signaled and a
1437 		 * later seqno has already been submitted. Assign a stub fence
1438 		 * so that the eventfd still gets signaled below.
1439 		 */
1440 		fence = dma_fence_get_stub();
1441 	}
1442 
1443 	list_del_init(&entry->node);
1444 	entry->fence = fence;
1445 
1446 	if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1447 		eventfd_signal(entry->ev_fd_ctx);
1448 		syncobj_eventfd_entry_free(entry);
1449 	} else {
1450 		ret = dma_fence_add_callback(fence, &entry->fence_cb,
1451 					     syncobj_eventfd_entry_fence_func);
1452 		if (ret == -ENOENT) {
1453 			eventfd_signal(entry->ev_fd_ctx);
1454 			syncobj_eventfd_entry_free(entry);
1455 		}
1456 	}
1457 }
1458 
1459 int
drm_syncobj_eventfd_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1460 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1461 			  struct drm_file *file_private)
1462 {
1463 	struct drm_syncobj_eventfd *args = data;
1464 	struct drm_syncobj *syncobj;
1465 	struct eventfd_ctx *ev_fd_ctx;
1466 	struct syncobj_eventfd_entry *entry;
1467 	int ret;
1468 
1469 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1470 		return -EOPNOTSUPP;
1471 
1472 	if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1473 		return -EINVAL;
1474 
1475 	if (args->pad)
1476 		return -EINVAL;
1477 
1478 	syncobj = drm_syncobj_find(file_private, args->handle);
1479 	if (!syncobj)
1480 		return -ENOENT;
1481 
1482 	ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1483 	if (IS_ERR(ev_fd_ctx)) {
1484 		ret = PTR_ERR(ev_fd_ctx);
1485 		goto err_fdget;
1486 	}
1487 
1488 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1489 	if (!entry) {
1490 		ret = -ENOMEM;
1491 		goto err_kzalloc;
1492 	}
1493 	entry->syncobj = syncobj;
1494 	entry->ev_fd_ctx = ev_fd_ctx;
1495 	entry->point = args->point;
1496 	entry->flags = args->flags;
1497 
1498 	drm_syncobj_add_eventfd(syncobj, entry);
1499 	drm_syncobj_put(syncobj);
1500 
1501 	return 0;
1502 
1503 err_kzalloc:
1504 	eventfd_ctx_put(ev_fd_ctx);
1505 err_fdget:
1506 	drm_syncobj_put(syncobj);
1507 	return ret;
1508 }
1509 
1510 int
drm_syncobj_reset_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1511 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1512 			struct drm_file *file_private)
1513 {
1514 	struct drm_syncobj_array *args = data;
1515 	struct drm_syncobj **syncobjs;
1516 	uint32_t i;
1517 	int ret;
1518 
1519 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1520 		return -EOPNOTSUPP;
1521 
1522 	if (args->pad != 0)
1523 		return -EINVAL;
1524 
1525 	if (args->count_handles == 0)
1526 		return -EINVAL;
1527 
1528 	ret = drm_syncobj_array_find(file_private,
1529 				     u64_to_user_ptr(args->handles),
1530 				     args->count_handles,
1531 				     &syncobjs);
1532 	if (ret < 0)
1533 		return ret;
1534 
1535 	for (i = 0; i < args->count_handles; i++)
1536 		drm_syncobj_replace_fence(syncobjs[i], NULL);
1537 
1538 	drm_syncobj_array_free(syncobjs, args->count_handles);
1539 
1540 	return 0;
1541 }
1542 
1543 int
drm_syncobj_signal_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1544 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1545 			 struct drm_file *file_private)
1546 {
1547 	struct drm_syncobj_array *args = data;
1548 	struct drm_syncobj **syncobjs;
1549 	uint32_t i;
1550 	int ret;
1551 
1552 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1553 		return -EOPNOTSUPP;
1554 
1555 	if (args->pad != 0)
1556 		return -EINVAL;
1557 
1558 	if (args->count_handles == 0)
1559 		return -EINVAL;
1560 
1561 	ret = drm_syncobj_array_find(file_private,
1562 				     u64_to_user_ptr(args->handles),
1563 				     args->count_handles,
1564 				     &syncobjs);
1565 	if (ret < 0)
1566 		return ret;
1567 
1568 	for (i = 0; i < args->count_handles; i++) {
1569 		ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1570 		if (ret < 0)
1571 			break;
1572 	}
1573 
1574 	drm_syncobj_array_free(syncobjs, args->count_handles);
1575 
1576 	return ret;
1577 }
1578 
1579 int
drm_syncobj_timeline_signal_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1580 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1581 				  struct drm_file *file_private)
1582 {
1583 	struct drm_syncobj_timeline_array *args = data;
1584 	struct drm_syncobj **syncobjs;
1585 	struct dma_fence_chain **chains;
1586 	uint64_t *points;
1587 	uint32_t i, j;
1588 	int ret;
1589 
1590 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1591 		return -EOPNOTSUPP;
1592 
1593 	if (args->flags != 0)
1594 		return -EINVAL;
1595 
1596 	if (args->count_handles == 0)
1597 		return -EINVAL;
1598 
1599 	ret = drm_syncobj_array_find(file_private,
1600 				     u64_to_user_ptr(args->handles),
1601 				     args->count_handles,
1602 				     &syncobjs);
1603 	if (ret < 0)
1604 		return ret;
1605 
1606 	points = kmalloc_array(args->count_handles, sizeof(*points),
1607 			       GFP_KERNEL);
1608 	if (!points) {
1609 		ret = -ENOMEM;
1610 		goto out;
1611 	}
1612 	if (!u64_to_user_ptr(args->points)) {
1613 		memset(points, 0, args->count_handles * sizeof(uint64_t));
1614 	} else if (copy_from_user(points, u64_to_user_ptr(args->points),
1615 				  sizeof(uint64_t) * args->count_handles)) {
1616 		ret = -EFAULT;
1617 		goto err_points;
1618 	}
1619 
1620 	chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1621 	if (!chains) {
1622 		ret = -ENOMEM;
1623 		goto err_points;
1624 	}
1625 	for (i = 0; i < args->count_handles; i++) {
1626 		chains[i] = dma_fence_chain_alloc();
1627 		if (!chains[i]) {
1628 			for (j = 0; j < i; j++)
1629 				dma_fence_chain_free(chains[j]);
1630 			ret = -ENOMEM;
1631 			goto err_chains;
1632 		}
1633 	}
1634 
1635 	for (i = 0; i < args->count_handles; i++) {
1636 		struct dma_fence *fence = dma_fence_get_stub();
1637 
1638 		drm_syncobj_add_point(syncobjs[i], chains[i],
1639 				      fence, points[i]);
1640 		dma_fence_put(fence);
1641 	}
1642 err_chains:
1643 	kfree(chains);
1644 err_points:
1645 	kfree(points);
1646 out:
1647 	drm_syncobj_array_free(syncobjs, args->count_handles);
1648 
1649 	return ret;
1650 }
1651 
drm_syncobj_query_ioctl(struct drm_device * dev,void * data,struct drm_file * file_private)1652 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1653 			    struct drm_file *file_private)
1654 {
1655 	struct drm_syncobj_timeline_array *args = data;
1656 	struct drm_syncobj **syncobjs;
1657 	uint64_t __user *points = u64_to_user_ptr(args->points);
1658 	uint32_t i;
1659 	int ret;
1660 
1661 	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1662 		return -EOPNOTSUPP;
1663 
1664 	if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1665 		return -EINVAL;
1666 
1667 	if (args->count_handles == 0)
1668 		return -EINVAL;
1669 
1670 	ret = drm_syncobj_array_find(file_private,
1671 				     u64_to_user_ptr(args->handles),
1672 				     args->count_handles,
1673 				     &syncobjs);
1674 	if (ret < 0)
1675 		return ret;
1676 
1677 	for (i = 0; i < args->count_handles; i++) {
1678 		struct dma_fence_chain *chain;
1679 		struct dma_fence *fence;
1680 		uint64_t point;
1681 
1682 		fence = drm_syncobj_fence_get(syncobjs[i]);
1683 		chain = to_dma_fence_chain(fence);
1684 		if (chain) {
1685 			struct dma_fence *iter, *last_signaled =
1686 				dma_fence_get(fence);
1687 
1688 			if (args->flags &
1689 			    DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1690 				point = fence->seqno;
1691 			} else {
1692 				dma_fence_chain_for_each(iter, fence) {
1693 					if (iter->context != fence->context) {
1694 						dma_fence_put(iter);
1695 						/* It is most likely that timeline has
1696 						* unorder points. */
1697 						break;
1698 					}
1699 					dma_fence_put(last_signaled);
1700 					last_signaled = dma_fence_get(iter);
1701 				}
1702 				point = dma_fence_is_signaled(last_signaled) ?
1703 					last_signaled->seqno :
1704 					to_dma_fence_chain(last_signaled)->prev_seqno;
1705 			}
1706 			dma_fence_put(last_signaled);
1707 		} else {
1708 			point = 0;
1709 		}
1710 		dma_fence_put(fence);
1711 		ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1712 		ret = ret ? -EFAULT : 0;
1713 		if (ret)
1714 			break;
1715 	}
1716 	drm_syncobj_array_free(syncobjs, args->count_handles);
1717 
1718 	return ret;
1719 }
1720