Lines Matching +full:active +full:-
2 * SPDX-License-Identifier: MIT
18 * Active refs memory management
21 * they idle (when we know the active requests are inactive) and allocate the
36 node_from_active(struct i915_active_fence *active) in node_from_active() argument
38 return container_of(active, struct active_node, base); in node_from_active()
41 #define take_preallocated_barriers(x) llist_del_all(&(x)->preallocated_barriers)
43 static inline bool is_barrier(const struct i915_active_fence *active) in is_barrier() argument
45 return IS_ERR(rcu_access_pointer(active->fence)); in is_barrier()
50 GEM_BUG_ON(!is_barrier(&node->base)); in barrier_to_ll()
51 return (struct llist_node *)&node->base.cb.node; in barrier_to_ll()
57 return (struct intel_engine_cs *)READ_ONCE(node->base.cb.node.prev); in __barrier_to_engine()
63 GEM_BUG_ON(!is_barrier(&node->base)); in barrier_to_engine()
79 return (void *)ref->active ?: (void *)ref->retire ?: (void *)ref; in active_debug_hint()
94 lockdep_assert_held(&ref->tree_lock); in debug_active_activate()
100 lockdep_assert_held(&ref->tree_lock); in debug_active_deactivate()
101 if (!atomic_read(&ref->count)) /* after the last dec */ in debug_active_deactivate()
134 /* return the unused nodes to our slabcache -- flushing the allocator */ in __active_retire()
135 if (!atomic_dec_and_lock_irqsave(&ref->count, &ref->tree_lock, flags)) in __active_retire()
138 GEM_BUG_ON(rcu_access_pointer(ref->excl.fence)); in __active_retire()
142 if (!ref->cache) in __active_retire()
143 ref->cache = fetch_node(ref->tree.rb_node); in __active_retire()
146 if (ref->cache) { in __active_retire()
148 rb_erase(&ref->cache->node, &ref->tree); in __active_retire()
149 root = ref->tree; in __active_retire()
152 rb_link_node(&ref->cache->node, NULL, &ref->tree.rb_node); in __active_retire()
153 rb_insert_color(&ref->cache->node, &ref->tree); in __active_retire()
154 GEM_BUG_ON(ref->tree.rb_node != &ref->cache->node); in __active_retire()
157 ref->cache->timeline = 0; /* needs cmpxchg(u64) */ in __active_retire()
160 spin_unlock_irqrestore(&ref->tree_lock, flags); in __active_retire()
163 if (ref->retire) in __active_retire()
164 ref->retire(ref); in __active_retire()
171 GEM_BUG_ON(i915_active_fence_isset(&it->base)); in __active_retire()
181 GEM_BUG_ON(!atomic_read(&ref->count)); in active_work()
182 if (atomic_add_unless(&ref->count, -1, 1)) in active_work()
191 GEM_BUG_ON(!atomic_read(&ref->count)); in active_retire()
192 if (atomic_add_unless(&ref->count, -1, 1)) in active_retire()
195 if (ref->flags & I915_ACTIVE_RETIRE_SLEEPS) { in active_retire()
196 queue_work(system_unbound_wq, &ref->work); in active_retire()
204 __active_fence_slot(struct i915_active_fence *active) in __active_fence_slot() argument
206 return (struct dma_fence ** __force)&active->fence; in __active_fence_slot()
212 struct i915_active_fence *active = in active_fence_cb() local
213 container_of(cb, typeof(*active), cb); in active_fence_cb()
215 return try_cmpxchg(__active_fence_slot(active), &fence, NULL); in active_fence_cb()
222 active_retire(container_of(cb, struct active_node, base.cb)->ref); in node_retire()
245 it = READ_ONCE(ref->cache); in __active_lookup()
247 u64 cached = READ_ONCE(it->timeline); in __active_lookup()
256 * If the value is already non-zero, some other thread has in __active_lookup()
263 if (!cached && !cmpxchg64(&it->timeline, 0, idx)) in __active_lookup()
269 /* While active, the tree can only be built; not destroyed */ in __active_lookup()
272 it = fetch_node(ref->tree.rb_node); in __active_lookup()
274 if (it->timeline < idx) { in __active_lookup()
275 it = fetch_node(it->node.rb_right); in __active_lookup()
276 } else if (it->timeline > idx) { in __active_lookup()
277 it = fetch_node(it->node.rb_left); in __active_lookup()
279 WRITE_ONCE(ref->cache, it); in __active_lookup()
296 return &node->base; in active_instance()
298 spin_lock_irq(&ref->tree_lock); in active_instance()
302 p = &ref->tree.rb_node; in active_instance()
307 if (node->timeline == idx) in active_instance()
310 if (node->timeline < idx) in active_instance()
311 p = &parent->rb_right; in active_instance()
313 p = &parent->rb_left; in active_instance()
324 __i915_active_fence_init(&node->base, NULL, node_retire); in active_instance()
325 node->ref = ref; in active_instance()
326 node->timeline = idx; in active_instance()
328 rb_link_node(&node->node, parent, p); in active_instance()
329 rb_insert_color(&node->node, &ref->tree); in active_instance()
332 WRITE_ONCE(ref->cache, node); in active_instance()
333 spin_unlock_irq(&ref->tree_lock); in active_instance()
335 return &node->base; in active_instance()
339 int (*active)(struct i915_active *ref), in __i915_active_init()
347 ref->flags = flags; in __i915_active_init()
348 ref->active = active; in __i915_active_init()
349 ref->retire = retire; in __i915_active_init()
351 spin_lock_init(&ref->tree_lock); in __i915_active_init()
352 ref->tree = RB_ROOT; in __i915_active_init()
353 ref->cache = NULL; in __i915_active_init()
355 init_llist_head(&ref->preallocated_barriers); in __i915_active_init()
356 atomic_set(&ref->count, 0); in __i915_active_init()
357 __mutex_init(&ref->mutex, "i915_active", mkey); in __i915_active_init()
358 __i915_active_fence_init(&ref->excl, NULL, excl_retire); in __i915_active_init()
359 INIT_WORK(&ref->work, active_work); in __i915_active_init()
361 lockdep_init_map(&ref->work.lockdep_map, "i915_active.work", wkey, 0); in __i915_active_init()
373 GEM_BUG_ON(node->timeline != engine->kernel_context->timeline->fence_context); in ____active_del_barrier()
378 * else may be manipulating the engine->barrier_tasks, in in ____active_del_barrier()
386 * any of the tasks, but we will try again on the next -- and since in ____active_del_barrier()
390 llist_for_each_safe(pos, next, llist_del_all(&engine->barrier_tasks)) { in ____active_del_barrier()
396 pos->next = head; in ____active_del_barrier()
402 llist_add_batch(head, tail, &engine->barrier_tasks); in ____active_del_barrier()
414 replace_barrier(struct i915_active *ref, struct i915_active_fence *active) in replace_barrier() argument
416 if (!is_barrier(active)) /* proto-node used by our idle barrier? */ in replace_barrier()
421 * we can use it to substitute for the pending idle-barrer in replace_barrier()
424 return __active_del_barrier(ref, node_from_active(active)); in replace_barrier()
429 u64 idx = i915_request_timeline(rq)->fence_context; in i915_active_add_request()
430 struct dma_fence *fence = &rq->fence; in i915_active_add_request()
431 struct i915_active_fence *active; in i915_active_add_request() local
440 active = active_instance(ref, idx); in i915_active_add_request()
441 if (!active) { in i915_active_add_request()
442 err = -ENOMEM; in i915_active_add_request()
446 if (replace_barrier(ref, active)) { in i915_active_add_request()
447 RCU_INIT_POINTER(active->fence, NULL); in i915_active_add_request()
448 atomic_dec(&ref->count); in i915_active_add_request()
450 } while (unlikely(is_barrier(active))); in i915_active_add_request()
452 fence = __i915_active_fence_set(active, fence); in i915_active_add_request()
465 struct i915_active_fence *active, in __i915_active_set_fence() argument
470 if (replace_barrier(ref, active)) { in __i915_active_set_fence()
471 RCU_INIT_POINTER(active->fence, fence); in __i915_active_set_fence()
475 prev = __i915_active_fence_set(active, fence); in __i915_active_set_fence()
486 return __i915_active_set_fence(ref, &ref->excl, f); in i915_active_set_exclusive()
492 return atomic_add_unless(&ref->count, 1, 0); in i915_active_acquire_if_busy()
497 spin_lock_irq(&ref->tree_lock); /* __active_retire() */ in __i915_active_activate()
498 if (!atomic_fetch_inc(&ref->count)) in __i915_active_activate()
500 spin_unlock_irq(&ref->tree_lock); in __i915_active_activate()
510 if (!ref->active) { in i915_active_acquire()
515 err = mutex_lock_interruptible(&ref->mutex); in i915_active_acquire()
520 err = ref->active(ref); in i915_active_acquire()
525 mutex_unlock(&ref->mutex); in i915_active_acquire()
536 static void enable_signaling(struct i915_active_fence *active) in enable_signaling() argument
540 if (unlikely(is_barrier(active))) in enable_signaling()
543 fence = i915_active_fence_get(active); in enable_signaling()
555 if (likely(!is_barrier(&it->base))) in flush_barrier()
560 if (!is_barrier(&it->base)) in flush_barrier()
571 enable_signaling(&ref->excl); in flush_lazy_signals()
572 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { in flush_lazy_signals()
577 enable_signaling(&it->base); in flush_lazy_signals()
587 /* Any fence added after the wait begins will not be auto-signaled */ in __i915_active_wait()
598 return -EINTR; in __i915_active_wait()
602 * After the wait is complete, the caller may free the active. in __i915_active_wait()
605 flush_work(&ref->work); in __i915_active_wait()
609 static int __await_active(struct i915_active_fence *active, in __await_active() argument
615 if (is_barrier(active)) /* XXX flush the barrier? */ in __await_active()
618 fence = i915_active_fence_get(active); in __await_active()
641 if (i915_active_is_idle(wb->ref)) { in barrier_wake()
642 list_del(&wq->entry); in barrier_wake()
643 i915_sw_fence_complete(wq->private); in barrier_wake()
656 return -ENOMEM; in __await_barrier()
661 return -EINVAL; in __await_barrier()
664 wb->base.flags = 0; in __await_barrier()
665 wb->base.func = barrier_wake; in __await_barrier()
666 wb->base.private = fence; in __await_barrier()
667 wb->ref = ref; in __await_barrier()
669 add_wait_queue(__var_waitqueue(ref), &wb->base); in __await_barrier()
684 rcu_access_pointer(ref->excl.fence)) { in await_active()
685 err = __await_active(&ref->excl, fn, arg); in await_active()
693 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { in await_active()
694 err = __await_active(&it->base, fn, arg); in await_active()
724 return await_active(ref, flags, rq_await_fence, rq, &rq->submit); in i915_request_await_active()
743 GEM_BUG_ON(atomic_read(&ref->count)); in i915_active_fini()
744 GEM_BUG_ON(work_pending(&ref->work)); in i915_active_fini()
745 mutex_destroy(&ref->mutex); in i915_active_fini()
747 if (ref->cache) in i915_active_fini()
748 kmem_cache_free(slab_cache, ref->cache); in i915_active_fini()
753 return node->timeline == idx && !i915_active_fence_isset(&node->base); in is_idle_barrier()
760 if (RB_EMPTY_ROOT(&ref->tree)) in reuse_idle_barrier()
767 * i915_active, due to overlapping active phases there is likely a in reuse_idle_barrier()
772 if (ref->cache && is_idle_barrier(ref->cache, idx)) { in reuse_idle_barrier()
773 p = &ref->cache->node; in reuse_idle_barrier()
778 p = ref->tree.rb_node; in reuse_idle_barrier()
787 if (node->timeline < idx) in reuse_idle_barrier()
788 p = READ_ONCE(p->rb_right); in reuse_idle_barrier()
790 p = READ_ONCE(p->rb_left); in reuse_idle_barrier()
795 * kernel_context. Walk the rb_tree in-order to see if there were in reuse_idle_barrier()
796 * any idle-barriers on this timeline that we missed, or just use in reuse_idle_barrier()
804 if (node->timeline > idx) in reuse_idle_barrier()
807 if (node->timeline < idx) in reuse_idle_barrier()
822 if (is_barrier(&node->base) && in reuse_idle_barrier()
830 spin_lock_irq(&ref->tree_lock); in reuse_idle_barrier()
831 rb_erase(p, &ref->tree); /* Hide from waits and sibling allocations */ in reuse_idle_barrier()
832 if (p == &ref->cache->node) in reuse_idle_barrier()
833 WRITE_ONCE(ref->cache, NULL); in reuse_idle_barrier()
834 spin_unlock_irq(&ref->tree_lock); in reuse_idle_barrier()
842 intel_engine_mask_t tmp, mask = engine->mask; in i915_active_acquire_preallocate_barrier()
844 struct intel_gt *gt = engine->gt; in i915_active_acquire_preallocate_barrier()
849 while (!llist_empty(&ref->preallocated_barriers)) in i915_active_acquire_preallocate_barrier()
860 u64 idx = engine->kernel_context->timeline->fence_context; in i915_active_acquire_preallocate_barrier()
872 RCU_INIT_POINTER(node->base.fence, NULL); in i915_active_acquire_preallocate_barrier()
873 node->base.cb.func = node_retire; in i915_active_acquire_preallocate_barrier()
874 node->timeline = idx; in i915_active_acquire_preallocate_barrier()
875 node->ref = ref; in i915_active_acquire_preallocate_barrier()
878 if (!i915_active_fence_isset(&node->base)) { in i915_active_acquire_preallocate_barrier()
880 * Mark this as being *our* unconnected proto-node. in i915_active_acquire_preallocate_barrier()
884 * request to indicate this is an idle-barrier node in i915_active_acquire_preallocate_barrier()
888 RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN)); in i915_active_acquire_preallocate_barrier()
889 node->base.cb.node.prev = (void *)engine; in i915_active_acquire_preallocate_barrier()
892 GEM_BUG_ON(rcu_access_pointer(node->base.fence) != ERR_PTR(-EAGAIN)); in i915_active_acquire_preallocate_barrier()
896 first->next = prev; in i915_active_acquire_preallocate_barrier()
902 GEM_BUG_ON(!llist_empty(&ref->preallocated_barriers)); in i915_active_acquire_preallocate_barrier()
903 llist_add_batch(first, last, &ref->preallocated_barriers); in i915_active_acquire_preallocate_barrier()
911 first = first->next; in i915_active_acquire_preallocate_barrier()
913 atomic_dec(&ref->count); in i915_active_acquire_preallocate_barrier()
918 return -ENOMEM; in i915_active_acquire_preallocate_barrier()
930 * i915_active rbtree, but only as proto-nodes. They will be in i915_active_acquire_barrier()
939 spin_lock_irqsave_nested(&ref->tree_lock, flags, in i915_active_acquire_barrier()
942 p = &ref->tree.rb_node; in i915_active_acquire_barrier()
949 if (it->timeline < node->timeline) in i915_active_acquire_barrier()
950 p = &parent->rb_right; in i915_active_acquire_barrier()
952 p = &parent->rb_left; in i915_active_acquire_barrier()
954 rb_link_node(&node->node, parent, p); in i915_active_acquire_barrier()
955 rb_insert_color(&node->node, &ref->tree); in i915_active_acquire_barrier()
956 spin_unlock_irqrestore(&ref->tree_lock, flags); in i915_active_acquire_barrier()
959 llist_add(barrier_to_ll(node), &engine->barrier_tasks); in i915_active_acquire_barrier()
966 return __active_fence_slot(&barrier_from_ll(node)->base); in ll_to_fence_slot()
971 struct intel_engine_cs *engine = rq->engine; in i915_request_add_active_barriers()
975 GEM_BUG_ON(!intel_context_is_barrier(rq->context)); in i915_request_add_active_barriers()
977 GEM_BUG_ON(i915_request_timeline(rq) != engine->kernel_context->timeline); in i915_request_add_active_barriers()
979 node = llist_del_all(&engine->barrier_tasks); in i915_request_add_active_barriers()
983 * Attach the list of proto-fences to the in-flight request such in i915_request_add_active_barriers()
987 spin_lock_irqsave(&rq->lock, flags); in i915_request_add_active_barriers()
990 smp_store_mb(*ll_to_fence_slot(node), &rq->fence); in i915_request_add_active_barriers()
991 list_add_tail((struct list_head *)node, &rq->fence.cb_list); in i915_request_add_active_barriers()
993 spin_unlock_irqrestore(&rq->lock, flags); in i915_request_add_active_barriers()
997 * __i915_active_fence_set: Update the last active fence along its timeline
998 * @active: the active tracker
1001 * Records the new @fence as the last active fence along its timeline in
1002 * this active tracker, moving the tracking callbacks from the previous
1010 __i915_active_fence_set(struct i915_active_fence *active, in __i915_active_fence_set() argument
1021 * while tracked under a different active tracker. Combined with i915 in __i915_active_fence_set()
1026 * As a countermeasure, we try to get a reference to the active->fence in __i915_active_fence_set()
1031 prev = i915_active_fence_get(active); in __i915_active_fence_set()
1035 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); in __i915_active_fence_set()
1039 * C already resident as the active->fence. in __i915_active_fence_set()
1046 * nesting rules for the fence->lock; the inner lock is always the in __i915_active_fence_set()
1049 spin_lock_irqsave(fence->lock, flags); in __i915_active_fence_set()
1051 spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING); in __i915_active_fence_set()
1058 * active->fence. Meanwhile, B follows the same path as A. in __i915_active_fence_set()
1060 * active->fence, locks it as soon as A completes, and possibly in __i915_active_fence_set()
1063 while (cmpxchg(__active_fence_slot(active), prev, fence) != prev) { in __i915_active_fence_set()
1065 spin_unlock(prev->lock); in __i915_active_fence_set()
1068 spin_unlock_irqrestore(fence->lock, flags); in __i915_active_fence_set()
1070 prev = i915_active_fence_get(active); in __i915_active_fence_set()
1073 spin_lock_irqsave(fence->lock, flags); in __i915_active_fence_set()
1075 spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING); in __i915_active_fence_set()
1088 * itself -- remembering that it needs to wait on A before executing. in __i915_active_fence_set()
1091 __list_del_entry(&active->cb.node); in __i915_active_fence_set()
1092 spin_unlock(prev->lock); /* serialise with prev->cb_list */ in __i915_active_fence_set()
1094 list_add_tail(&active->cb.node, &fence->cb_list); in __i915_active_fence_set()
1095 spin_unlock_irqrestore(fence->lock, flags); in __i915_active_fence_set()
1100 int i915_active_fence_set(struct i915_active_fence *active, in i915_active_fence_set() argument
1106 /* Must maintain timeline ordering wrt previous active requests */ in i915_active_fence_set()
1107 fence = __i915_active_fence_set(active, &rq->fence); in i915_active_fence_set()
1130 kref_get(&aa->ref); in i915_active_get()
1131 return &aa->base; in i915_active_get()
1138 i915_active_fini(&aa->base); in auto_release()
1146 kref_put(&aa->ref, auto_release); in i915_active_put()
1168 kref_init(&aa->ref); in i915_active_create()
1169 i915_active_init(&aa->base, auto_active, auto_retire, 0); in i915_active_create()
1171 return &aa->base; in i915_active_create()
1187 return -ENOMEM; in i915_active_module_init()