1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * dma-fence-util: misc functions for dma_fence objects 4 * 5 * Copyright (C) 2022 Advanced Micro Devices, Inc. 6 * Authors: 7 * Christian König <christian.koenig@amd.com> 8 */ 9 10 #include <linux/dma-fence.h> 11 #include <linux/dma-fence-array.h> 12 #include <linux/dma-fence-chain.h> 13 #include <linux/dma-fence-unwrap.h> 14 #include <linux/slab.h> 15 #include <linux/sort.h> 16 17 /* Internal helper to start new array iteration, don't use directly */ 18 static struct dma_fence * 19 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor) 20 { 21 cursor->array = dma_fence_chain_contained(cursor->chain); 22 cursor->index = 0; 23 return dma_fence_array_first(cursor->array); 24 } 25 26 /** 27 * dma_fence_unwrap_first - return the first fence from fence containers 28 * @head: the entrypoint into the containers 29 * @cursor: current position inside the containers 30 * 31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the 32 * first fence. 33 */ 34 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, 35 struct dma_fence_unwrap *cursor) 36 { 37 cursor->chain = dma_fence_get(head); 38 return __dma_fence_unwrap_array(cursor); 39 } 40 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first); 41 42 /** 43 * dma_fence_unwrap_next - return the next fence from a fence containers 44 * @cursor: current position inside the containers 45 * 46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return 47 * the next fence from them. 48 */ 49 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) 50 { 51 struct dma_fence *tmp; 52 53 ++cursor->index; 54 tmp = dma_fence_array_next(cursor->array, cursor->index); 55 if (tmp) 56 return tmp; 57 58 cursor->chain = dma_fence_chain_walk(cursor->chain); 59 return __dma_fence_unwrap_array(cursor); 60 } 61 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); 62 63 64 static int fence_cmp(const void *_a, const void *_b) 65 { 66 struct dma_fence *a = *(struct dma_fence **)_a; 67 struct dma_fence *b = *(struct dma_fence **)_b; 68 69 if (a->context < b->context) 70 return -1; 71 else if (a->context > b->context) 72 return 1; 73 74 if (dma_fence_is_later(b, a)) 75 return 1; 76 else if (dma_fence_is_later(a, b)) 77 return -1; 78 79 return 0; 80 } 81 82 /** 83 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers 84 * @fences: Array of dma_fence pointers to be deduplicated 85 * @num_fences: Number of entries in the @fences array 86 * 87 * Sorts the input array by context, then removes duplicate 88 * fences with the same context, keeping only the most recent one. 89 * 90 * The array is modified in-place and unreferenced duplicate fences are released 91 * via dma_fence_put(). The function returns the new number of fences after 92 * deduplication. 93 * 94 * Return: Number of unique fences remaining in the array. 95 */ 96 int dma_fence_dedup_array(struct dma_fence **fences, int num_fences) 97 { 98 int i, j; 99 100 sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL); 101 102 /* 103 * Only keep the most recent fence for each context. 104 */ 105 j = 0; 106 for (i = 1; i < num_fences; i++) { 107 if (fences[i]->context == fences[j]->context) 108 dma_fence_put(fences[i]); 109 else 110 fences[++j] = fences[i]; 111 } 112 113 return ++j; 114 } 115 EXPORT_SYMBOL_GPL(dma_fence_dedup_array); 116 117 /* Implementation for the dma_fence_merge() marco, don't use directly */ 118 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, 119 struct dma_fence **fences, 120 struct dma_fence_unwrap *iter) 121 { 122 struct dma_fence *tmp, *unsignaled = NULL, **array; 123 struct dma_fence_array *result; 124 ktime_t timestamp; 125 int i, count; 126 127 count = 0; 128 timestamp = ns_to_ktime(0); 129 for (i = 0; i < num_fences; ++i) { 130 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { 131 if (!dma_fence_is_signaled(tmp)) { 132 dma_fence_put(unsignaled); 133 unsignaled = dma_fence_get(tmp); 134 ++count; 135 } else { 136 ktime_t t = dma_fence_timestamp(tmp); 137 138 if (ktime_after(t, timestamp)) 139 timestamp = t; 140 } 141 } 142 } 143 144 /* 145 * If we couldn't find a pending fence just return a private signaled 146 * fence with the timestamp of the last signaled one. 147 * 148 * Or if there was a single unsignaled fence left we can return it 149 * directly and early since that is a major path on many workloads. 150 */ 151 if (count == 0) 152 return dma_fence_allocate_private_stub(timestamp); 153 else if (count == 1) 154 return unsignaled; 155 156 dma_fence_put(unsignaled); 157 158 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL); 159 if (!array) 160 return NULL; 161 162 count = 0; 163 for (i = 0; i < num_fences; ++i) { 164 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { 165 if (!dma_fence_is_signaled(tmp)) { 166 array[count++] = dma_fence_get(tmp); 167 } else { 168 ktime_t t = dma_fence_timestamp(tmp); 169 170 if (ktime_after(t, timestamp)) 171 timestamp = t; 172 } 173 } 174 } 175 176 if (count == 0 || count == 1) 177 goto return_fastpath; 178 179 count = dma_fence_dedup_array(array, count); 180 181 if (count > 1) { 182 result = dma_fence_array_create(count, array, 183 dma_fence_context_alloc(1), 184 1, false); 185 if (!result) { 186 for (i = 0; i < count; i++) 187 dma_fence_put(array[i]); 188 tmp = NULL; 189 goto return_tmp; 190 } 191 return &result->base; 192 } 193 194 return_fastpath: 195 if (count == 0) 196 tmp = dma_fence_allocate_private_stub(timestamp); 197 else 198 tmp = array[0]; 199 200 return_tmp: 201 kfree(array); 202 return tmp; 203 } 204 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge); 205