1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * fence-chain: chain fences together in a timeline 4 * 5 * Copyright (C) 2018 Advanced Micro Devices, Inc. 6 * Authors: 7 * Christian König <christian.koenig@amd.com> 8 */ 9 10 #include <linux/dma-fence-chain.h> 11 12 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); 13 14 /** 15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence 16 * @chain: chain node to get the previous node from 17 * 18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the 19 * chain node. 20 */ 21 static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) 22 { 23 struct dma_fence *prev; 24 25 rcu_read_lock(); 26 prev = dma_fence_get_rcu_safe(&chain->prev); 27 rcu_read_unlock(); 28 return prev; 29 } 30 31 /** 32 * dma_fence_chain_walk - chain walking function 33 * @fence: current chain node 34 * 35 * Walk the chain to the next node. Returns the next fence or NULL if we are at 36 * the end of the chain. Garbage collects chain nodes which are already 37 * signaled. 38 */ 39 struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) 40 { 41 struct dma_fence_chain *chain, *prev_chain; 42 struct dma_fence *prev, *replacement, *tmp; 43 44 chain = to_dma_fence_chain(fence); 45 if (!chain) { 46 dma_fence_put(fence); 47 return NULL; 48 } 49 50 while ((prev = dma_fence_chain_get_prev(chain))) { 51 52 prev_chain = to_dma_fence_chain(prev); 53 if (prev_chain) { 54 if (!dma_fence_is_signaled(prev_chain->fence)) 55 break; 56 57 replacement = dma_fence_chain_get_prev(prev_chain); 58 } else { 59 if (!dma_fence_is_signaled(prev)) 60 break; 61 62 replacement = NULL; 63 } 64 65 tmp = cmpxchg((struct dma_fence __force **)&chain->prev, 66 prev, replacement); 67 if (tmp == prev) 68 dma_fence_put(tmp); 69 else 70 dma_fence_put(replacement); 71 dma_fence_put(prev); 72 } 73 74 dma_fence_put(fence); 75 return prev; 76 } 77 EXPORT_SYMBOL(dma_fence_chain_walk); 78 79 /** 80 * dma_fence_chain_find_seqno - find fence chain node by seqno 81 * @pfence: pointer to the chain node where to start 82 * @seqno: the sequence number to search for 83 * 84 * Advance the fence pointer to the chain node which will signal this sequence 85 * number. If no sequence number is provided then this is a no-op. 86 * 87 * Returns EINVAL if the fence is not a chain node or the sequence number has 88 * not yet advanced far enough. 89 */ 90 int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) 91 { 92 struct dma_fence_chain *chain; 93 94 if (!seqno) 95 return 0; 96 97 chain = to_dma_fence_chain(*pfence); 98 if (!chain || chain->base.seqno < seqno) 99 return -EINVAL; 100 101 dma_fence_chain_for_each(*pfence, &chain->base) { 102 if ((*pfence)->context != chain->base.context || 103 to_dma_fence_chain(*pfence)->prev_seqno < seqno) 104 break; 105 } 106 dma_fence_put(&chain->base); 107 108 return 0; 109 } 110 EXPORT_SYMBOL(dma_fence_chain_find_seqno); 111 112 static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) 113 { 114 return "dma_fence_chain"; 115 } 116 117 static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) 118 { 119 return "unbound"; 120 } 121 122 static void dma_fence_chain_irq_work(struct irq_work *work) 123 { 124 struct dma_fence_chain *chain; 125 126 chain = container_of(work, typeof(*chain), work); 127 128 /* Try to rearm the callback */ 129 if (!dma_fence_chain_enable_signaling(&chain->base)) 130 /* Ok, we are done. No more unsignaled fences left */ 131 dma_fence_signal(&chain->base); 132 dma_fence_put(&chain->base); 133 } 134 135 static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) 136 { 137 struct dma_fence_chain *chain; 138 139 chain = container_of(cb, typeof(*chain), cb); 140 irq_work_queue(&chain->work); 141 dma_fence_put(f); 142 } 143 144 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) 145 { 146 struct dma_fence_chain *head = to_dma_fence_chain(fence); 147 148 dma_fence_get(&head->base); 149 dma_fence_chain_for_each(fence, &head->base) { 150 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 151 struct dma_fence *f = chain ? chain->fence : fence; 152 153 dma_fence_get(f); 154 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { 155 dma_fence_put(fence); 156 return true; 157 } 158 dma_fence_put(f); 159 } 160 dma_fence_put(&head->base); 161 return false; 162 } 163 164 static bool dma_fence_chain_signaled(struct dma_fence *fence) 165 { 166 dma_fence_chain_for_each(fence, fence) { 167 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 168 struct dma_fence *f = chain ? chain->fence : fence; 169 170 if (!dma_fence_is_signaled(f)) { 171 dma_fence_put(fence); 172 return false; 173 } 174 } 175 176 return true; 177 } 178 179 static void dma_fence_chain_release(struct dma_fence *fence) 180 { 181 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 182 struct dma_fence *prev; 183 184 /* Manually unlink the chain as much as possible to avoid recursion 185 * and potential stack overflow. 186 */ 187 while ((prev = rcu_dereference_protected(chain->prev, true))) { 188 struct dma_fence_chain *prev_chain; 189 190 if (kref_read(&prev->refcount) > 1) 191 break; 192 193 prev_chain = to_dma_fence_chain(prev); 194 if (!prev_chain) 195 break; 196 197 /* No need for atomic operations since we hold the last 198 * reference to prev_chain. 199 */ 200 chain->prev = prev_chain->prev; 201 RCU_INIT_POINTER(prev_chain->prev, NULL); 202 dma_fence_put(prev); 203 } 204 dma_fence_put(prev); 205 206 dma_fence_put(chain->fence); 207 dma_fence_free(fence); 208 } 209 210 const struct dma_fence_ops dma_fence_chain_ops = { 211 .use_64bit_seqno = true, 212 .get_driver_name = dma_fence_chain_get_driver_name, 213 .get_timeline_name = dma_fence_chain_get_timeline_name, 214 .enable_signaling = dma_fence_chain_enable_signaling, 215 .signaled = dma_fence_chain_signaled, 216 .release = dma_fence_chain_release, 217 }; 218 EXPORT_SYMBOL(dma_fence_chain_ops); 219 220 /** 221 * dma_fence_chain_init - initialize a fence chain 222 * @chain: the chain node to initialize 223 * @prev: the previous fence 224 * @fence: the current fence 225 * @seqno: the sequence number to use for the fence chain 226 * 227 * Initialize a new chain node and either start a new chain or add the node to 228 * the existing chain of the previous fence. 229 */ 230 void dma_fence_chain_init(struct dma_fence_chain *chain, 231 struct dma_fence *prev, 232 struct dma_fence *fence, 233 uint64_t seqno) 234 { 235 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); 236 uint64_t context; 237 238 spin_lock_init(&chain->lock); 239 rcu_assign_pointer(chain->prev, prev); 240 chain->fence = fence; 241 chain->prev_seqno = 0; 242 init_irq_work(&chain->work, dma_fence_chain_irq_work); 243 244 /* Try to reuse the context of the previous chain node. */ 245 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { 246 context = prev->context; 247 chain->prev_seqno = prev->seqno; 248 } else { 249 context = dma_fence_context_alloc(1); 250 /* Make sure that we always have a valid sequence number. */ 251 if (prev_chain) 252 seqno = max(prev->seqno, seqno); 253 } 254 255 dma_fence_init(&chain->base, &dma_fence_chain_ops, 256 &chain->lock, context, seqno); 257 } 258 EXPORT_SYMBOL(dma_fence_chain_init); 259