3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
31  * fence context counter: each execution context should have its own
32  * fence context, this allows checking if fences belong to the same
45  * A fence is initialized using dma_fence_init() and completed using
60  *   fence to be updated.
69  * DOC: fence cross-driver contract
95  *   This means any code required for fence completion cannot acquire a
100  *   callbacks. This means any code required for fence completion cannot
105  *   for fence completion cannot allocate memory with GFP_NOFS or GFP_NOIO.
114 static const char *dma_fence_stub_get_name(struct dma_fence *fence)  in dma_fence_stub_get_name()  argument
138  * dma_fence_get_stub - return a signaled fence
140  * Return a stub fence which is already signaled. The fence's timestamp
150  * dma_fence_allocate_private_stub - return a private, signaled fence
151  * @timestamp: timestamp when the fence was signaled
153  * Return a newly allocated and signaled stub fence.
157 	struct dma_fence *fence;  in dma_fence_allocate_private_stub()  local
159 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);  in dma_fence_allocate_private_stub()
160 	if (fence == NULL)  in dma_fence_allocate_private_stub()
163 	dma_fence_init(fence,  in dma_fence_allocate_private_stub()
169 		&fence->flags);  in dma_fence_allocate_private_stub()
171 	dma_fence_signal_timestamp(fence, timestamp);  in dma_fence_allocate_private_stub()
173 	return fence;  in dma_fence_allocate_private_stub()
178  * dma_fence_context_alloc - allocate an array of fence contexts
181  * This function will return the first index of the number of fence contexts
182  * allocated.  The fence context is used for setting &dma_fence.context to a
193  * DOC: fence signalling annotation
233  * prevents it from signalling the fence the previous thread is stuck waiting
254  *   point where a fence is accessible to other threads, to the point where
287  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
318  * dma_fence_end_signalling - end a critical DMA fence signalling section
348  * dma_fence_signal_timestamp_locked - signal completion of a fence
349  * @fence: the fence to signal
350  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
352  * Signal completion for software callbacks on a fence, this will unblock
354  * dma_fence_add_callback(). Can be called multiple times, but since a fence
356  * only be effective the first time. Set the timestamp provided as the fence
362  * Returns 0 on success and a negative error value when @fence has been
365 int dma_fence_signal_timestamp_locked(struct dma_fence *fence,  in dma_fence_signal_timestamp_locked()  argument
371 	lockdep_assert_held(fence->lock);  in dma_fence_signal_timestamp_locked()
374 				      &fence->flags)))  in dma_fence_signal_timestamp_locked()
378 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_timestamp_locked()
380 	fence->timestamp = timestamp;  in dma_fence_signal_timestamp_locked()
381 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_timestamp_locked()
382 	trace_dma_fence_signaled(fence);  in dma_fence_signal_timestamp_locked()
386 		cur->func(fence, cur);  in dma_fence_signal_timestamp_locked()
394  * dma_fence_signal_timestamp - signal completion of a fence
395  * @fence: the fence to signal
396  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
398  * Signal completion for software callbacks on a fence, this will unblock
400  * dma_fence_add_callback(). Can be called multiple times, but since a fence
402  * only be effective the first time. Set the timestamp provided as the fence
405  * Returns 0 on success and a negative error value when @fence has been
408 int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)  in dma_fence_signal_timestamp()  argument
413 	if (WARN_ON(!fence))  in dma_fence_signal_timestamp()
416 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal_timestamp()
417 	ret = dma_fence_signal_timestamp_locked(fence, timestamp);  in dma_fence_signal_timestamp()
418 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal_timestamp()
425  * dma_fence_signal_locked - signal completion of a fence
426  * @fence: the fence to signal
428  * Signal completion for software callbacks on a fence, this will unblock
430  * dma_fence_add_callback(). Can be called multiple times, but since a fence
437  * Returns 0 on success and a negative error value when @fence has been
440 int dma_fence_signal_locked(struct dma_fence *fence)  in dma_fence_signal_locked()  argument
442 	return dma_fence_signal_timestamp_locked(fence, ktime_get());  in dma_fence_signal_locked()
447  * dma_fence_signal - signal completion of a fence
448  * @fence: the fence to signal
450  * Signal completion for software callbacks on a fence, this will unblock
452  * dma_fence_add_callback(). Can be called multiple times, but since a fence
456  * Returns 0 on success and a negative error value when @fence has been
459 int dma_fence_signal(struct dma_fence *fence)  in dma_fence_signal()  argument
465 	if (WARN_ON(!fence))  in dma_fence_signal()
470 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
471 	ret = dma_fence_signal_timestamp_locked(fence, ktime_get());  in dma_fence_signal()
472 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
481  * dma_fence_wait_timeout - sleep until the fence gets signaled
483  * @fence: the fence to wait on
491  * Performs a synchronous wait on this fence. It is assumed the caller
493  * holds a reference to the fence, otherwise the fence might be
499 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_wait_timeout()  argument
510 	dma_fence_enable_sw_signaling(fence);  in dma_fence_wait_timeout()
514 		trace_dma_fence_wait_start(fence);  in dma_fence_wait_timeout()
517 	if (fence->ops->wait)  in dma_fence_wait_timeout()
518 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
520 		ret = dma_fence_default_wait(fence, intr, timeout);  in dma_fence_wait_timeout()
523 		trace_dma_fence_wait_end(fence);  in dma_fence_wait_timeout()
539 	struct dma_fence *fence =  in dma_fence_release()  local
543 	trace_dma_fence_destroy(fence);  in dma_fence_release()
545 	if (!list_empty(&fence->cb_list) &&  in dma_fence_release()
546 	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_release()
551 		driver = dma_fence_driver_name(fence);  in dma_fence_release()
552 		timeline = dma_fence_timeline_name(fence);  in dma_fence_release()
555 		     "Fence %s:%s:%llx:%llx released with pending signals!\n",  in dma_fence_release()
557 		     fence->context, fence->seqno);  in dma_fence_release()
566 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
567 		fence->error = -EDEADLK;  in dma_fence_release()
568 		dma_fence_signal_locked(fence);  in dma_fence_release()
569 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
574 	if (fence->ops->release)  in dma_fence_release()
575 		fence->ops->release(fence);  in dma_fence_release()
577 		dma_fence_free(fence);  in dma_fence_release()
583  * @fence: fence to release
586  * kfree_rcu() on @fence.
588 void dma_fence_free(struct dma_fence *fence)  in dma_fence_free()  argument
590 	kfree_rcu(fence, rcu);  in dma_fence_free()
594 static bool __dma_fence_enable_signaling(struct dma_fence *fence)  in __dma_fence_enable_signaling()  argument
598 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
601 				   &fence->flags);  in __dma_fence_enable_signaling()
603 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
606 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
607 		trace_dma_fence_enable_signal(fence);  in __dma_fence_enable_signaling()
609 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
610 			dma_fence_signal_locked(fence);  in __dma_fence_enable_signaling()
619  * dma_fence_enable_sw_signaling - enable signaling on fence
620  * @fence: the fence to enable
622  * This will request for sw signaling to be enabled, to make the fence
626 void dma_fence_enable_sw_signaling(struct dma_fence *fence)  in dma_fence_enable_sw_signaling()  argument
630 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
631 	__dma_fence_enable_signaling(fence);  in dma_fence_enable_sw_signaling()
632 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
637  * dma_fence_add_callback - add a callback to be called when the fence
639  * @fence: the fence to wait on
643  * Add a software callback to the fence. The caller should keep a reference to
644  * the fence.
648  * to a fence, but a callback can only be registered to one fence at a time.
650  * If fence is already signaled, this function will return -ENOENT (and
655  * Returns 0 in case of success, -ENOENT if the fence is already signaled
658 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
664 	if (WARN_ON(!fence || !func))  in dma_fence_add_callback()
667 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
672 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
674 	if (__dma_fence_enable_signaling(fence)) {  in dma_fence_add_callback()
676 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
682 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
690  * @fence: the dma_fence to query
693  * condition on a signaled fence. See dma_fence_get_status_locked() for more
696  * Returns 0 if the fence has not yet been signaled, 1 if the fence has
698  * if the fence has been completed in err.
700 int dma_fence_get_status(struct dma_fence *fence)  in dma_fence_get_status()  argument
705 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
706 	status = dma_fence_get_status_locked(fence);  in dma_fence_get_status()
707 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
715  * @fence: the fence to wait on
718  * Remove a previously queued callback from the fence. This function returns
719  * true if the callback is successfully removed, or false if the fence has
726  * with a reference held to the fence.
728  * Behaviour is undefined if @cb has not been added to @fence using
732 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
737 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
743 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
755 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
764  * dma_fence_default_wait - default sleep until the fence gets signaled
766  * @fence: the fence to wait on
772  * returned if the fence is already signaled for consistency with other
776 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_default_wait()  argument
782 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
784 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
799 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
801 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
806 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
810 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
820 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
832 		struct dma_fence *fence = fences[i];  in dma_fence_test_signaled_any()  local
833 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
843  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
849  * @idx: used to store the first signaled fence index, meaningful only on
852  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
856  * Synchronous waits for the first fence in the array to be signaled. The
858  * fence might be freed before return, resulting in undefined behavior.
891 		struct dma_fence *fence = fences[i];  in dma_fence_wait_any_timeout()  local
894 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
896 			/* This fence is already signaled */  in dma_fence_wait_any_timeout()
961  * hint (Ie. forward progress cannot be made until this fence is signaled).
963  * Multiple deadlines may be set on a given fence, even in parallel.  See the
966  * The deadline hint is just that, a hint.  The driver that created the fence
972  * dma_fence_set_deadline - set desired fence-wait deadline hint
973  * @fence:    the fence that is to be waited on
974  * @deadline: the time by which the waiter hopes for the fence to be
977  * Give the fence signaler a hint about an upcoming deadline, such as
978  * vblank, by which point the waiter would prefer the fence to be
979  * signaled by.  This is intended to give feedback to the fence signaler
981  * if a periodic vblank deadline is approaching but the fence is not
984 void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)  in dma_fence_set_deadline()  argument
986 	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))  in dma_fence_set_deadline()
987 		fence->ops->set_deadline(fence, deadline);  in dma_fence_set_deadline()
992  * dma_fence_describe - Dump fence description into seq_file
993  * @fence: the fence to describe
996  * Dump a textual description of the fence and it's state into the seq_file.
998 void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq)  in dma_fence_describe()  argument
1006 	if (!dma_fence_is_signaled(fence)) {  in dma_fence_describe()
1007 		timeline = dma_fence_timeline_name(fence);  in dma_fence_describe()
1008 		driver = dma_fence_driver_name(fence);  in dma_fence_describe()
1013 		   fence->context, fence->seqno, timeline, driver,  in dma_fence_describe()
1021 __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in __dma_fence_init()  argument
1027 	kref_init(&fence->refcount);  in __dma_fence_init()
1028 	fence->ops = ops;  in __dma_fence_init()
1029 	INIT_LIST_HEAD(&fence->cb_list);  in __dma_fence_init()
1030 	fence->lock = lock;  in __dma_fence_init()
1031 	fence->context = context;  in __dma_fence_init()
1032 	fence->seqno = seqno;  in __dma_fence_init()
1033 	fence->flags = flags;  in __dma_fence_init()
1034 	fence->error = 0;  in __dma_fence_init()
1036 	trace_dma_fence_init(fence);  in __dma_fence_init()
1040  * dma_fence_init - Initialize a custom fence.
1041  * @fence: the fence to initialize
1042  * @ops: the dma_fence_ops for operations on this fence
1043  * @lock: the irqsafe spinlock to use for locking this fence
1044  * @context: the execution context this fence is run on
1047  * Initializes an allocated fence, the caller doesn't have to keep its
1048  * refcount after committing with this fence, but it will need to hold a
1052  * to check which fence is later by simply using dma_fence_later().
1055 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init()  argument
1058 	__dma_fence_init(fence, ops, lock, context, seqno, 0UL);  in dma_fence_init()
1063  * dma_fence_init64 - Initialize a custom fence with 64-bit seqno support.
1064  * @fence: the fence to initialize
1065  * @ops: the dma_fence_ops for operations on this fence
1066  * @lock: the irqsafe spinlock to use for locking this fence
1067  * @context: the execution context this fence is run on
1070  * Initializes an allocated fence, the caller doesn't have to keep its
1071  * refcount after committing with this fence, but it will need to hold a
1075  * to check which fence is later by simply using dma_fence_later().
1078 dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init64()  argument
1081 	__dma_fence_init(fence, ops, lock, context, seqno,  in dma_fence_init64()
1088  * @fence: the fence to query
1090  * Returns a driver name backing the dma-fence implementation.
1093  * Dma-fence contract stipulates that access to driver provided data (data not
1096  * after the fence has been signalled. Drivers are allowed to free that data,
1100  * between signalling the fence and freeing said data.
1106 const char __rcu *dma_fence_driver_name(struct dma_fence *fence)  in dma_fence_driver_name()  argument
1111 	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_driver_name()
1112 		return fence->ops->get_driver_name(fence);  in dma_fence_driver_name()
1120  * @fence: the fence to query
1122  * Returns a timeline name provided by the dma-fence implementation.
1125  * Dma-fence contract stipulates that access to driver provided data (data not
1128  * after the fence has been signalled. Drivers are allowed to free that data,
1132  * between signalling the fence and freeing said data.
1138 const char __rcu *dma_fence_timeline_name(struct dma_fence *fence)  in dma_fence_timeline_name()  argument
1143 	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_timeline_name()
1144 		return fence->ops->get_timeline_name(fence);  in dma_fence_timeline_name()