1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * dma-fence-util: misc functions for dma_fence objects 4 * 5 * Copyright (C) 2022 Advanced Micro Devices, Inc. 6 * Authors: 7 * Christian König <christian.koenig@amd.com> 8 */ 9 10 #include <linux/dma-fence.h> 11 #include <linux/dma-fence-array.h> 12 #include <linux/dma-fence-chain.h> 13 #include <linux/dma-fence-unwrap.h> 14 #include <linux/slab.h> 15 #include <linux/sort.h> 16 17 /* Internal helper to start new array iteration, don't use directly */ 18 static struct dma_fence * 19 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor) 20 { 21 cursor->array = dma_fence_chain_contained(cursor->chain); 22 cursor->index = 0; 23 return dma_fence_array_first(cursor->array); 24 } 25 26 /** 27 * dma_fence_unwrap_first - return the first fence from fence containers 28 * @head: the entrypoint into the containers 29 * @cursor: current position inside the containers 30 * 31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the 32 * first fence. 33 */ 34 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, 35 struct dma_fence_unwrap *cursor) 36 { 37 cursor->chain = dma_fence_get(head); 38 return __dma_fence_unwrap_array(cursor); 39 } 40 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first); 41 42 /** 43 * dma_fence_unwrap_next - return the next fence from a fence containers 44 * @cursor: current position inside the containers 45 * 46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return 47 * the next fence from them. 48 */ 49 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) 50 { 51 struct dma_fence *tmp; 52 53 ++cursor->index; 54 tmp = dma_fence_array_next(cursor->array, cursor->index); 55 if (tmp) 56 return tmp; 57 58 cursor->chain = dma_fence_chain_walk(cursor->chain); 59 return __dma_fence_unwrap_array(cursor); 60 } 61 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); 62 63 64 static int fence_cmp(const void *_a, const void *_b) 65 { 66 struct dma_fence *a = *(struct dma_fence **)_a; 67 struct dma_fence *b = *(struct dma_fence **)_b; 68 69 if (a->context < b->context) 70 return -1; 71 else if (a->context > b->context) 72 return 1; 73 74 if (dma_fence_is_later(b, a)) 75 return 1; 76 else if (dma_fence_is_later(a, b)) 77 return -1; 78 79 return 0; 80 } 81 82 /* Implementation for the dma_fence_merge() marco, don't use directly */ 83 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, 84 struct dma_fence **fences, 85 struct dma_fence_unwrap *iter) 86 { 87 struct dma_fence *tmp, *unsignaled = NULL, **array; 88 struct dma_fence_array *result; 89 ktime_t timestamp; 90 int i, j, count; 91 92 count = 0; 93 timestamp = ns_to_ktime(0); 94 for (i = 0; i < num_fences; ++i) { 95 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { 96 if (!dma_fence_is_signaled(tmp)) { 97 dma_fence_put(unsignaled); 98 unsignaled = dma_fence_get(tmp); 99 ++count; 100 } else { 101 ktime_t t = dma_fence_timestamp(tmp); 102 103 if (ktime_after(t, timestamp)) 104 timestamp = t; 105 } 106 } 107 } 108 109 /* 110 * If we couldn't find a pending fence just return a private signaled 111 * fence with the timestamp of the last signaled one. 112 * 113 * Or if there was a single unsignaled fence left we can return it 114 * directly and early since that is a major path on many workloads. 115 */ 116 if (count == 0) 117 return dma_fence_allocate_private_stub(timestamp); 118 else if (count == 1) 119 return unsignaled; 120 121 dma_fence_put(unsignaled); 122 123 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL); 124 if (!array) 125 return NULL; 126 127 count = 0; 128 for (i = 0; i < num_fences; ++i) { 129 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { 130 if (!dma_fence_is_signaled(tmp)) { 131 array[count++] = dma_fence_get(tmp); 132 } else { 133 ktime_t t = dma_fence_timestamp(tmp); 134 135 if (ktime_after(t, timestamp)) 136 timestamp = t; 137 } 138 } 139 } 140 141 if (count == 0 || count == 1) 142 goto return_fastpath; 143 144 sort(array, count, sizeof(*array), fence_cmp, NULL); 145 146 /* 147 * Only keep the most recent fence for each context. 148 */ 149 j = 0; 150 for (i = 1; i < count; i++) { 151 if (array[i]->context == array[j]->context) 152 dma_fence_put(array[i]); 153 else 154 array[++j] = array[i]; 155 } 156 count = ++j; 157 158 if (count > 1) { 159 result = dma_fence_array_create(count, array, 160 dma_fence_context_alloc(1), 161 1, false); 162 if (!result) { 163 for (i = 0; i < count; i++) 164 dma_fence_put(array[i]); 165 tmp = NULL; 166 goto return_tmp; 167 } 168 return &result->base; 169 } 170 171 return_fastpath: 172 if (count == 0) 173 tmp = dma_fence_allocate_private_stub(timestamp); 174 else 175 tmp = array[0]; 176 177 return_tmp: 178 kfree(array); 179 return tmp; 180 } 181 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge); 182