1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <linux/dma-fence.h> 7 #include <linux/interval_tree_generic.h> 8 #include <linux/slab.h> 9 10 #include "xe_macros.h" 11 #include "xe_range_fence.h" 12 13 #define XE_RANGE_TREE_START(_node) ((_node)->start) 14 #define XE_RANGE_TREE_LAST(_node) ((_node)->last) 15 16 INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last, 17 XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static, 18 xe_range_fence_tree); 19 20 static void 21 xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb) 22 { 23 struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb); 24 struct xe_range_fence_tree *tree = rfence->tree; 25 26 llist_add(&rfence->link, &tree->list); 27 } 28 29 static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree) 30 { 31 struct llist_node *node = llist_del_all(&tree->list); 32 struct xe_range_fence *rfence, *next; 33 34 llist_for_each_entry_safe(rfence, next, node, link) { 35 xe_range_fence_tree_remove(rfence, &tree->root); 36 dma_fence_put(rfence->fence); 37 kfree(rfence); 38 } 39 40 return !!node; 41 } 42 43 /** 44 * xe_range_fence_insert() - range fence insert 45 * @tree: range fence tree to insert intoi 46 * @rfence: range fence 47 * @ops: range fence ops 48 * @start: start address of range fence 49 * @last: last address of range fence 50 * @fence: dma fence which signals range fence can be removed + freed 51 * 52 * Return: 0 on success, non-zero on failure 53 */ 54 int xe_range_fence_insert(struct xe_range_fence_tree *tree, 55 struct xe_range_fence *rfence, 56 const struct xe_range_fence_ops *ops, 57 u64 start, u64 last, struct dma_fence *fence) 58 { 59 int err = 0; 60 61 __xe_range_fence_tree_cleanup(tree); 62 63 if (dma_fence_is_signaled(fence)) 64 goto free; 65 66 rfence->ops = ops; 67 rfence->start = start; 68 rfence->last = last; 69 rfence->tree = tree; 70 rfence->fence = dma_fence_get(fence); 71 err = dma_fence_add_callback(fence, &rfence->cb, 72 xe_range_fence_signal_notify); 73 if (err == -ENOENT) { 74 dma_fence_put(fence); 75 err = 0; 76 goto free; 77 } else if (err == 0) { 78 xe_range_fence_tree_insert(rfence, &tree->root); 79 return 0; 80 } else { 81 dma_fence_put(fence); 82 } 83 84 free: 85 if (ops->free) 86 ops->free(rfence); 87 88 return err; 89 } 90 91 static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree) 92 { 93 struct xe_range_fence *rfence; 94 bool retry = true; 95 96 rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX); 97 while (rfence) { 98 /* Should be ok with the minimalistic callback */ 99 if (dma_fence_remove_callback(rfence->fence, &rfence->cb)) 100 llist_add(&rfence->link, &tree->list); 101 rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX); 102 } 103 104 while (retry) 105 retry = __xe_range_fence_tree_cleanup(tree); 106 } 107 108 /** 109 * xe_range_fence_tree_init() - Init range fence tree 110 * @tree: range fence tree 111 */ 112 void xe_range_fence_tree_init(struct xe_range_fence_tree *tree) 113 { 114 memset(tree, 0, sizeof(*tree)); 115 } 116 117 /** 118 * xe_range_fence_tree_fini() - Fini range fence tree 119 * @tree: range fence tree 120 */ 121 void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree) 122 { 123 xe_range_fence_tree_remove_all(tree); 124 XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root)); 125 } 126 127 /** 128 * xe_range_fence_tree_first() - range fence tree iterator first 129 * @tree: range fence tree 130 * @start: start address of range fence 131 * @last: last address of range fence 132 * 133 * Return: first range fence found in range or NULL 134 */ 135 struct xe_range_fence * 136 xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start, 137 u64 last) 138 { 139 return xe_range_fence_tree_iter_first(&tree->root, start, last); 140 } 141 142 /** 143 * xe_range_fence_tree_next() - range fence tree iterator next 144 * @rfence: current range fence 145 * @start: start address of range fence 146 * @last: last address of range fence 147 * 148 * Return: next range fence found in range or NULL 149 */ 150 struct xe_range_fence * 151 xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last) 152 { 153 return xe_range_fence_tree_iter_next(rfence, start, last); 154 } 155 156 static void xe_range_fence_free(struct xe_range_fence *rfence) 157 { 158 kfree(rfence); 159 } 160 161 const struct xe_range_fence_ops xe_range_fence_kfree_ops = { 162 .free = xe_range_fence_free, 163 }; 164