54 	struct i915_sw_fence *fence;  member
60 static const char *i915_fence_get_driver_name(struct dma_fence *fence)  in i915_fence_get_driver_name()  argument
62 	return dev_name(to_request(fence)->i915->drm.dev);  in i915_fence_get_driver_name()
65 static const char *i915_fence_get_timeline_name(struct dma_fence *fence)  in i915_fence_get_timeline_name()  argument
78 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_fence_get_timeline_name()
81 	ctx = i915_request_gem_context(to_request(fence));  in i915_fence_get_timeline_name()
88 static bool i915_fence_signaled(struct dma_fence *fence)  in i915_fence_signaled()  argument
90 	return i915_request_completed(to_request(fence));  in i915_fence_signaled()
93 static bool i915_fence_enable_signaling(struct dma_fence *fence)  in i915_fence_enable_signaling()  argument
95 	return i915_request_enable_breadcrumb(to_request(fence));  in i915_fence_enable_signaling()
98 static signed long i915_fence_wait(struct dma_fence *fence,  in i915_fence_wait()  argument
102 	return i915_request_wait_timeout(to_request(fence),  in i915_fence_wait()
112 static void i915_fence_release(struct dma_fence *fence)  in i915_fence_release()  argument
114 	struct i915_request *rq = to_request(fence);  in i915_fence_release()
186 	i915_sw_fence_complete(cb->fence);  in irq_execute_cb()
387 		dma_fence_signal_locked(&rq->fence);  in i915_request_retire()
391 	if (test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags))  in i915_request_retire()
490 			inflight = i915_seqno_passed(rq->fence.seqno,  in __request_in_flight()
491 						     signal->fence.seqno);  in __request_in_flight()
514 	cb->fence = &rq->submit;  in __await_execution()
515 	i915_sw_fence_await(cb->fence);  in __await_execution()
554 	GEM_BUG_ON(!fatal_error(rq->fence.error));  in __i915_request_skip()
559 	RQ_TRACE(rq, "error: %d\n", rq->fence.error);  in __i915_request_skip()
579 	old = READ_ONCE(rq->fence.error);  in i915_request_set_error_once()
583 	} while (!try_cmpxchg(&rq->fence.error, &old, error));  in i915_request_set_error_once()
638 	if (unlikely(fatal_error(request->fence.error)))  in __i915_request_submit()
672 	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));  in __i915_request_submit()
675 	clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);  in __i915_request_submit()
676 	set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);  in __i915_request_submit()
691 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_submit()
730 	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));  in __i915_request_unsubmit()
731 	clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);  in __i915_request_unsubmit()
732 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_unsubmit()
766 	set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);  in i915_request_cancel()
772 submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)  in submit_notify()  argument
775 		container_of(fence, typeof(*request), submit);  in submit_notify()
781 		if (unlikely(fence->error))  in submit_notify()
782 			i915_request_set_error_once(request, fence->error);  in submit_notify()
808 semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)  in semaphore_notify()  argument
810 	struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);  in semaphore_notify()
955 	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock,  in __i915_request_create()
1061 	struct dma_fence *fence;  in i915_request_await_start()  local
1078 	fence = NULL;  in i915_request_await_start()
1108 		fence = &prev->fence;  in i915_request_await_start()
1111 	if (!fence)  in i915_request_await_start()
1115 	if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))  in i915_request_await_start()
1117 						    fence, 0,  in i915_request_await_start()
1119 	dma_fence_put(fence);  in i915_request_await_start()
1236 	if (__emit_semaphore_wait(to, from, from->fence.seqno))  in emit_semaphore_wait()
1244 					     &from->fence, 0,  in emit_semaphore_wait()
1249 					  struct dma_fence *fence)  in intel_timeline_sync_has_start()  argument
1252 					      fence->context,  in intel_timeline_sync_has_start()
1253 					      fence->seqno - 1);  in intel_timeline_sync_has_start()
1257 					 const struct dma_fence *fence)  in intel_timeline_sync_set_start()  argument
1259 	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);  in intel_timeline_sync_set_start()
1277 					  &from->fence))  in __i915_request_await_execution()
1316 		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);  in __i915_request_await_execution()
1331 					     &from->fence);  in __i915_request_await_execution()
1348 __i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)  in __i915_request_await_external()  argument
1351 	return i915_sw_fence_await_dma_fence(&rq->submit, fence,  in __i915_request_await_external()
1353 									fence->context),  in __i915_request_await_external()
1358 i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)  in i915_request_await_external()  argument
1363 	if (!to_dma_fence_chain(fence))  in i915_request_await_external()
1364 		return __i915_request_await_external(rq, fence);  in i915_request_await_external()
1366 	dma_fence_chain_for_each(iter, fence) {  in i915_request_await_external()
1369 		if (!dma_fence_is_i915(chain->fence)) {  in i915_request_await_external()
1374 		err = i915_request_await_dma_fence(rq, chain->fence);  in i915_request_await_external()
1404 			     struct dma_fence *fence)  in i915_request_await_execution()  argument
1406 	struct dma_fence **child = &fence;  in i915_request_await_execution()
1410 	if (dma_fence_is_array(fence)) {  in i915_request_await_execution()
1411 		struct dma_fence_array *array = to_dma_fence_array(fence);  in i915_request_await_execution()
1421 		fence = *child++;  in i915_request_await_execution()
1422 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_request_await_execution()
1425 		if (fence->context == rq->fence.context)  in i915_request_await_execution()
1433 		if (dma_fence_is_i915(fence)) {  in i915_request_await_execution()
1434 			if (is_same_parallel_context(rq, to_request(fence)))  in i915_request_await_execution()
1437 							     to_request(fence));  in i915_request_await_execution()
1439 			ret = i915_request_await_external(rq, fence);  in i915_request_await_execution()
1476 		i915_sw_fence_set_error_once(&to->submit, from->fence.error);  in i915_request_await_request()
1500 i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)  in i915_request_await_dma_fence()  argument
1502 	struct dma_fence **child = &fence;  in i915_request_await_dma_fence()
1514 	if (dma_fence_is_array(fence)) {  in i915_request_await_dma_fence()
1515 		struct dma_fence_array *array = to_dma_fence_array(fence);  in i915_request_await_dma_fence()
1523 		fence = *child++;  in i915_request_await_dma_fence()
1524 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_request_await_dma_fence()
1532 		if (fence->context == rq->fence.context)  in i915_request_await_dma_fence()
1536 		if (fence->context &&  in i915_request_await_dma_fence()
1538 						 fence))  in i915_request_await_dma_fence()
1541 		if (dma_fence_is_i915(fence)) {  in i915_request_await_dma_fence()
1542 			if (is_same_parallel_context(rq, to_request(fence)))  in i915_request_await_dma_fence()
1544 			ret = i915_request_await_request(rq, to_request(fence));  in i915_request_await_dma_fence()
1546 			ret = i915_request_await_external(rq, fence);  in i915_request_await_dma_fence()
1552 		if (fence->context)  in i915_request_await_dma_fence()
1554 						fence);  in i915_request_await_dma_fence()
1607 	struct dma_fence *fence;  in i915_request_await_object()  local
1611 				dma_resv_usage_rw(write), fence) {  in i915_request_await_object()
1612 		ret = i915_request_await_dma_fence(to, fence);  in i915_request_await_object()
1630 					     &huc->delayed_load.fence,  in i915_request_await_huc()
1666 						  &rq->fence));  in __i915_request_ensure_parallel_ordering()
1678 						  &rq->fence));  in __i915_request_ensure_ordering()
1693 			   i915_seqno_passed(prev->fence.seqno,  in __i915_request_ensure_ordering()
1694 					     rq->fence.seqno));  in __i915_request_ensure_ordering()
1702 							&prev->fence,  in __i915_request_ensure_ordering()
1776 	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);  in __i915_request_add_to_timeline()
1934 		if (dma_fence_is_signaled(&rq->fence))  in __i915_spin_request()
1954 static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)  in request_wait_wake()  argument
1991 	if (dma_fence_is_signaled(&rq->fence))  in i915_request_wait_timeout()
2050 	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))  in i915_request_wait_timeout()
2074 		if (dma_fence_is_signaled(&rq->fence))  in i915_request_wait_timeout()
2092 		dma_fence_remove_callback(&rq->fence, &wait.cb);  in i915_request_wait_timeout()
2173 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))  in fence_status()
2176 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))  in fence_status()
2187 	const char *name = rq->fence.ops->get_timeline_name((struct dma_fence *)&rq->fence);  in i915_request_show()
2226 		   rq->fence.context, rq->fence.seqno,  in i915_request_show()