1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <linux/dma-fence.h> 7 #include <linux/interval_tree_generic.h> 8 #include <linux/slab.h> 9 10 #include "xe_macros.h" 11 #include "xe_range_fence.h" 12 13 #define XE_RANGE_TREE_START(_node) ((_node)->start) 14 #define XE_RANGE_TREE_LAST(_node) ((_node)->last) 15 16 INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last, 17 XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static, 18 xe_range_fence_tree); 19 20 static void 21 xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb) 22 { 23 struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb); 24 struct xe_range_fence_tree *tree = rfence->tree; 25 26 llist_add(&rfence->link, &tree->list); 27 } 28 29 static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree) 30 { 31 struct llist_node *node = llist_del_all(&tree->list); 32 struct xe_range_fence *rfence, *next; 33 34 llist_for_each_entry_safe(rfence, next, node, link) { 35 xe_range_fence_tree_remove(rfence, &tree->root); 36 dma_fence_put(rfence->fence); 37 kfree(rfence); 38 } 39 40 return !!node; 41 } 42 43 /** 44 * xe_range_fence_insert() - range fence insert 45 * @tree: range fence tree to insert intoi 46 * @rfence: range fence 47 * @ops: range fence ops 48 * @start: start address of range fence 49 * @last: last address of range fence 50 * @fence: dma fence which signals range fence can be removed + freed 51 * 52 * Return: 0 on success, non-zero on failure 53 */ 54 int xe_range_fence_insert(struct xe_range_fence_tree *tree, 55 struct xe_range_fence *rfence, 56 const struct xe_range_fence_ops *ops, 57 u64 start, u64 last, struct dma_fence *fence) 58 { 59 int err = 0; 60 61 __xe_range_fence_tree_cleanup(tree); 62 63 if (dma_fence_is_signaled(fence)) 64 goto free; 65 66 rfence->ops = ops; 67 rfence->start = start; 68 rfence->last = last; 69 rfence->tree = tree; 70 rfence->fence = dma_fence_get(fence); 71 err = dma_fence_add_callback(fence, &rfence->cb, 72 xe_range_fence_signal_notify); 73 if (err == -ENOENT) { 74 dma_fence_put(fence); 75 err = 0; 76 goto free; 77 } else if (err == 0) { 78 xe_range_fence_tree_insert(rfence, &tree->root); 79 return 0; 80 } 81 82 free: 83 if (ops->free) 84 ops->free(rfence); 85 86 return err; 87 } 88 89 static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree) 90 { 91 struct xe_range_fence *rfence; 92 bool retry = true; 93 94 rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX); 95 while (rfence) { 96 /* Should be ok with the minimalistic callback */ 97 if (dma_fence_remove_callback(rfence->fence, &rfence->cb)) 98 llist_add(&rfence->link, &tree->list); 99 rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX); 100 } 101 102 while (retry) 103 retry = __xe_range_fence_tree_cleanup(tree); 104 } 105 106 /** 107 * xe_range_fence_tree_init() - Init range fence tree 108 * @tree: range fence tree 109 */ 110 void xe_range_fence_tree_init(struct xe_range_fence_tree *tree) 111 { 112 memset(tree, 0, sizeof(*tree)); 113 } 114 115 /** 116 * xe_range_fence_tree_fini() - Fini range fence tree 117 * @tree: range fence tree 118 */ 119 void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree) 120 { 121 xe_range_fence_tree_remove_all(tree); 122 XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root)); 123 } 124 125 /** 126 * xe_range_fence_tree_first() - range fence tree iterator first 127 * @tree: range fence tree 128 * @start: start address of range fence 129 * @last: last address of range fence 130 * 131 * Return: first range fence found in range or NULL 132 */ 133 struct xe_range_fence * 134 xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start, 135 u64 last) 136 { 137 return xe_range_fence_tree_iter_first(&tree->root, start, last); 138 } 139 140 /** 141 * xe_range_fence_tree_next() - range fence tree iterator next 142 * @rfence: current range fence 143 * @start: start address of range fence 144 * @last: last address of range fence 145 * 146 * Return: next range fence found in range or NULL 147 */ 148 struct xe_range_fence * 149 xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last) 150 { 151 return xe_range_fence_tree_iter_next(rfence, start, last); 152 } 153 154 const struct xe_range_fence_ops xe_range_fence_kfree_ops = { 155 .free = (void (*)(struct xe_range_fence *rfence)) kfree, 156 }; 157