1 /* 2 * fence-chain: chain fences together in a timeline 3 * 4 * Copyright (C) 2018 Advanced Micro Devices, Inc. 5 * Authors: 6 * Christian König <christian.koenig@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License version 2 as published by 10 * the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but WITHOUT 13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 * more details. 16 */ 17 18 #include <linux/dma-fence-chain.h> 19 20 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); 21 22 /** 23 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence 24 * @chain: chain node to get the previous node from 25 * 26 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the 27 * chain node. 28 */ 29 static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) 30 { 31 struct dma_fence *prev; 32 33 rcu_read_lock(); 34 prev = dma_fence_get_rcu_safe(&chain->prev); 35 rcu_read_unlock(); 36 return prev; 37 } 38 39 /** 40 * dma_fence_chain_walk - chain walking function 41 * @fence: current chain node 42 * 43 * Walk the chain to the next node. Returns the next fence or NULL if we are at 44 * the end of the chain. Garbage collects chain nodes which are already 45 * signaled. 46 */ 47 struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) 48 { 49 struct dma_fence_chain *chain, *prev_chain; 50 struct dma_fence *prev, *replacement, *tmp; 51 52 chain = to_dma_fence_chain(fence); 53 if (!chain) { 54 dma_fence_put(fence); 55 return NULL; 56 } 57 58 while ((prev = dma_fence_chain_get_prev(chain))) { 59 60 prev_chain = to_dma_fence_chain(prev); 61 if (prev_chain) { 62 if (!dma_fence_is_signaled(prev_chain->fence)) 63 break; 64 65 replacement = dma_fence_chain_get_prev(prev_chain); 66 } else { 67 if (!dma_fence_is_signaled(prev)) 68 break; 69 70 replacement = NULL; 71 } 72 73 tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement); 74 if (tmp == prev) 75 dma_fence_put(tmp); 76 else 77 dma_fence_put(replacement); 78 dma_fence_put(prev); 79 } 80 81 dma_fence_put(fence); 82 return prev; 83 } 84 EXPORT_SYMBOL(dma_fence_chain_walk); 85 86 /** 87 * dma_fence_chain_find_seqno - find fence chain node by seqno 88 * @pfence: pointer to the chain node where to start 89 * @seqno: the sequence number to search for 90 * 91 * Advance the fence pointer to the chain node which will signal this sequence 92 * number. If no sequence number is provided then this is a no-op. 93 * 94 * Returns EINVAL if the fence is not a chain node or the sequence number has 95 * not yet advanced far enough. 96 */ 97 int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) 98 { 99 struct dma_fence_chain *chain; 100 101 if (!seqno) 102 return 0; 103 104 chain = to_dma_fence_chain(*pfence); 105 if (!chain || chain->base.seqno < seqno) 106 return -EINVAL; 107 108 dma_fence_chain_for_each(*pfence, &chain->base) { 109 if ((*pfence)->context != chain->base.context || 110 to_dma_fence_chain(*pfence)->prev_seqno < seqno) 111 break; 112 } 113 dma_fence_put(&chain->base); 114 115 return 0; 116 } 117 EXPORT_SYMBOL(dma_fence_chain_find_seqno); 118 119 static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) 120 { 121 return "dma_fence_chain"; 122 } 123 124 static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) 125 { 126 return "unbound"; 127 } 128 129 static void dma_fence_chain_irq_work(struct irq_work *work) 130 { 131 struct dma_fence_chain *chain; 132 133 chain = container_of(work, typeof(*chain), work); 134 135 /* Try to rearm the callback */ 136 if (!dma_fence_chain_enable_signaling(&chain->base)) 137 /* Ok, we are done. No more unsignaled fences left */ 138 dma_fence_signal(&chain->base); 139 dma_fence_put(&chain->base); 140 } 141 142 static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) 143 { 144 struct dma_fence_chain *chain; 145 146 chain = container_of(cb, typeof(*chain), cb); 147 irq_work_queue(&chain->work); 148 dma_fence_put(f); 149 } 150 151 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) 152 { 153 struct dma_fence_chain *head = to_dma_fence_chain(fence); 154 155 dma_fence_get(&head->base); 156 dma_fence_chain_for_each(fence, &head->base) { 157 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 158 struct dma_fence *f = chain ? chain->fence : fence; 159 160 dma_fence_get(f); 161 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { 162 dma_fence_put(fence); 163 return true; 164 } 165 dma_fence_put(f); 166 } 167 dma_fence_put(&head->base); 168 return false; 169 } 170 171 static bool dma_fence_chain_signaled(struct dma_fence *fence) 172 { 173 dma_fence_chain_for_each(fence, fence) { 174 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 175 struct dma_fence *f = chain ? chain->fence : fence; 176 177 if (!dma_fence_is_signaled(f)) { 178 dma_fence_put(fence); 179 return false; 180 } 181 } 182 183 return true; 184 } 185 186 static void dma_fence_chain_release(struct dma_fence *fence) 187 { 188 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 189 190 dma_fence_put(rcu_dereference_protected(chain->prev, true)); 191 dma_fence_put(chain->fence); 192 dma_fence_free(fence); 193 } 194 195 const struct dma_fence_ops dma_fence_chain_ops = { 196 .use_64bit_seqno = true, 197 .get_driver_name = dma_fence_chain_get_driver_name, 198 .get_timeline_name = dma_fence_chain_get_timeline_name, 199 .enable_signaling = dma_fence_chain_enable_signaling, 200 .signaled = dma_fence_chain_signaled, 201 .release = dma_fence_chain_release, 202 }; 203 EXPORT_SYMBOL(dma_fence_chain_ops); 204 205 /** 206 * dma_fence_chain_init - initialize a fence chain 207 * @chain: the chain node to initialize 208 * @prev: the previous fence 209 * @fence: the current fence 210 * 211 * Initialize a new chain node and either start a new chain or add the node to 212 * the existing chain of the previous fence. 213 */ 214 void dma_fence_chain_init(struct dma_fence_chain *chain, 215 struct dma_fence *prev, 216 struct dma_fence *fence, 217 uint64_t seqno) 218 { 219 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); 220 uint64_t context; 221 222 spin_lock_init(&chain->lock); 223 rcu_assign_pointer(chain->prev, prev); 224 chain->fence = fence; 225 chain->prev_seqno = 0; 226 init_irq_work(&chain->work, dma_fence_chain_irq_work); 227 228 /* Try to reuse the context of the previous chain node. */ 229 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { 230 context = prev->context; 231 chain->prev_seqno = prev->seqno; 232 } else { 233 context = dma_fence_context_alloc(1); 234 /* Make sure that we always have a valid sequence number. */ 235 if (prev_chain) 236 seqno = max(prev->seqno, seqno); 237 } 238 239 dma_fence_init(&chain->base, &dma_fence_chain_ops, 240 &chain->lock, context, seqno); 241 } 242 EXPORT_SYMBOL(dma_fence_chain_init); 243