1 // SPDX-License-Identifier: GPL-2.0-only
3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
50  * Since the purposes of fences is to facilitate cross-device and
51  * cross-application synchronization, there's multiple ways to use one:
53  * - Individual fences can be exposed as a &sync_file, accessed as a file
58  * - Some subsystems also have their own explicit fencing primitives, like
62  * - Then there's also implicit fencing, where the synchronization points are
69  * DOC: fence cross-driver contract
77  *   further command submission and force complete all in-flight fences, e.g.
125  * dma_fence_get_stub - return a signaled fence
152  * dma_fence_allocate_private_stub - return a private, signaled fence
171 		&fence->flags);  in dma_fence_allocate_private_stub()
180  * dma_fence_context_alloc - allocate an array of fence contexts
200  * * It is a cross-driver contract, and therefore all drivers must follow the
202  *   and anything else significant for in-kernel interfaces. But it is also
203  *   impossible to test all drivers in a single machine, hence brute-force N vs.
219  *   this pulls in there is extremely thin wiggle-room for driver-specific
225  * Furthermore lockdep does not handle cross-release dependencies, which means
257  *   dma_fence_signal() is called. Un-annotated code can contain deadlock issues,
278  *   lockdep with the relevant hierarchy at boot-up. This means even just
289  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
312 	/* ... and non-recursive successful read_trylock */  in dma_fence_begin_signalling()
320  * dma_fence_end_signalling - end a critical DMA fence signalling section
350  * dma_fence_signal_timestamp_locked - signal completion of a fence
373 	lockdep_assert_held(fence->lock);  in dma_fence_signal_timestamp_locked()
376 				      &fence->flags)))  in dma_fence_signal_timestamp_locked()
377 		return -EINVAL;  in dma_fence_signal_timestamp_locked()
380 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_timestamp_locked()
382 	fence->timestamp = timestamp;  in dma_fence_signal_timestamp_locked()
383 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_timestamp_locked()
387 		INIT_LIST_HEAD(&cur->node);  in dma_fence_signal_timestamp_locked()
388 		cur->func(fence, cur);  in dma_fence_signal_timestamp_locked()
396  * dma_fence_signal_timestamp - signal completion of a fence
416 		return -EINVAL;  in dma_fence_signal_timestamp()
418 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal_timestamp()
420 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal_timestamp()
427  * dma_fence_signal_locked - signal completion of a fence
449  * dma_fence_signal - signal completion of a fence
468 		return -EINVAL;  in dma_fence_signal()
472 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
474 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
483  * dma_fence_wait_timeout - sleep until the fence gets signaled
489  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
494  * directly or indirectly (buf-mgr between reservation and committing)
506 		return -EINVAL;  in dma_fence_wait_timeout()
519 	if (fence->ops->wait)  in dma_fence_wait_timeout()
520 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
533  * dma_fence_release - default release function for fences
547 	if (!list_empty(&fence->cb_list) &&  in dma_fence_release()
548 	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_release()
559 		     fence->context, fence->seqno);  in dma_fence_release()
568 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
569 		fence->error = -EDEADLK;  in dma_fence_release()
571 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
576 	if (fence->ops->release)  in dma_fence_release()
577 		fence->ops->release(fence);  in dma_fence_release()
584  * dma_fence_free - default release function for &dma_fence.
600 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
603 				   &fence->flags);  in __dma_fence_enable_signaling()
605 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
608 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
611 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
621  * dma_fence_enable_sw_signaling - enable signaling on fence
632 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
634 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
639  * dma_fence_add_callback - add a callback to be called when the fence
652  * If fence is already signaled, this function will return -ENOENT (and
657  * Returns 0 in case of success, -ENOENT if the fence is already signaled
658  * and -EINVAL in case of error.
667 		return -EINVAL;  in dma_fence_add_callback()
669 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
670 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
671 		return -ENOENT;  in dma_fence_add_callback()
674 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
677 		cb->func = func;  in dma_fence_add_callback()
678 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
680 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
681 		ret = -ENOENT;  in dma_fence_add_callback()
684 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
691  * dma_fence_get_status - returns the status upon completion
707 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
709 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
716  * dma_fence_remove_callback - remove a callback from the signaling list
739 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
741 	ret = !list_empty(&cb->node);  in dma_fence_remove_callback()
743 		list_del_init(&cb->node);  in dma_fence_remove_callback()
745 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
762 	wake_up_state(wait->task, TASK_NORMAL);  in dma_fence_default_wait_cb()
766  * dma_fence_default_wait - default sleep until the fence gets signaled
772  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
784 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
786 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
790 		ret = -ERESTARTSYS;  in dma_fence_default_wait()
801 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
803 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
808 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
812 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
814 			ret = -ERESTARTSYS;  in dma_fence_default_wait()
822 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
835 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
845  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
852  *	positive return
854  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
873 		return -EINVAL;  in dma_fence_wait_any_timeout()
888 		ret = -ENOMEM;  in dma_fence_wait_any_timeout()
917 			ret = -ERESTARTSYS;  in dma_fence_wait_any_timeout()
923 	while (i-- > 0)  in dma_fence_wait_any_timeout()
938  * frequency that meets the requirements of the use-case, in order to minimize
942  * * Workloads that ping-pong between device and CPU, with alternating periods
948  *   buffered GPU rendering vs vblank sync'd page flipping.  In this scenario,
961  * the vblank based deadline for page-flipping, or the start of a compositor's
974  * dma_fence_set_deadline - set desired fence-wait deadline hint
988 	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))  in dma_fence_set_deadline()
989 		fence->ops->set_deadline(fence, deadline);  in dma_fence_set_deadline()
994  * dma_fence_describe - Dump fence description into seq_file
1013 		   fence->seqno,  in dma_fence_describe()
1025 	BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);  in __dma_fence_init()
1027 	kref_init(&fence->refcount);  in __dma_fence_init()
1028 	fence->ops = ops;  in __dma_fence_init()
1029 	INIT_LIST_HEAD(&fence->cb_list);  in __dma_fence_init()
1030 	fence->lock = lock;  in __dma_fence_init()
1031 	fence->context = context;  in __dma_fence_init()
1032 	fence->seqno = seqno;  in __dma_fence_init()
1033 	fence->flags = flags;  in __dma_fence_init()
1034 	fence->error = 0;  in __dma_fence_init()
1040  * dma_fence_init - Initialize a custom fence.
1063  * dma_fence_init64 - Initialize a custom fence with 64-bit seqno support.
1087  * dma_fence_driver_name - Access the driver name
1090  * Returns a driver name backing the dma-fence implementation.
1093  * Dma-fence contract stipulates that access to driver provided data (data not
1111 	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_driver_name()
1112 		return fence->ops->get_driver_name(fence);  in dma_fence_driver_name()
1114 		return "detached-driver";  in dma_fence_driver_name()
1119  * dma_fence_timeline_name - Access the timeline name
1122  * Returns a timeline name provided by the dma-fence implementation.
1125  * Dma-fence contract stipulates that access to driver provided data (data not
1143 	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_timeline_name()
1144 		return fence->ops->get_timeline_name(fence);  in dma_fence_timeline_name()
1146 		return "signaled-timeline";  in dma_fence_timeline_name()