1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * dma-fence-util: misc functions for dma_fence objects
4 *
5 * Copyright (C) 2022 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10 #include <linux/dma-fence.h>
11 #include <linux/dma-fence-array.h>
12 #include <linux/dma-fence-chain.h>
13 #include <linux/dma-fence-unwrap.h>
14 #include <linux/slab.h>
15 #include <linux/sort.h>
16
17 /* Internal helper to start new array iteration, don't use directly */
18 static struct dma_fence *
__dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)19 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
20 {
21 cursor->array = dma_fence_chain_contained(cursor->chain);
22 cursor->index = 0;
23 return dma_fence_array_first(cursor->array);
24 }
25
26 /**
27 * dma_fence_unwrap_first - return the first fence from fence containers
28 * @head: the entrypoint into the containers
29 * @cursor: current position inside the containers
30 *
31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
32 * first fence.
33 */
dma_fence_unwrap_first(struct dma_fence * head,struct dma_fence_unwrap * cursor)34 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
35 struct dma_fence_unwrap *cursor)
36 {
37 cursor->chain = dma_fence_get(head);
38 return __dma_fence_unwrap_array(cursor);
39 }
40 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
41
42 /**
43 * dma_fence_unwrap_next - return the next fence from a fence containers
44 * @cursor: current position inside the containers
45 *
46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
47 * the next fence from them.
48 */
dma_fence_unwrap_next(struct dma_fence_unwrap * cursor)49 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
50 {
51 struct dma_fence *tmp;
52
53 ++cursor->index;
54 tmp = dma_fence_array_next(cursor->array, cursor->index);
55 if (tmp)
56 return tmp;
57
58 cursor->chain = dma_fence_chain_walk(cursor->chain);
59 return __dma_fence_unwrap_array(cursor);
60 }
61 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
62
63
fence_cmp(const void * _a,const void * _b)64 static int fence_cmp(const void *_a, const void *_b)
65 {
66 struct dma_fence *a = *(struct dma_fence **)_a;
67 struct dma_fence *b = *(struct dma_fence **)_b;
68
69 if (a->context < b->context)
70 return -1;
71 else if (a->context > b->context)
72 return 1;
73
74 if (dma_fence_is_later(b, a))
75 return 1;
76 else if (dma_fence_is_later(a, b))
77 return -1;
78
79 return 0;
80 }
81
82 /**
83 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers
84 * @fences: Array of dma_fence pointers to be deduplicated
85 * @num_fences: Number of entries in the @fences array
86 *
87 * Sorts the input array by context, then removes duplicate
88 * fences with the same context, keeping only the most recent one.
89 *
90 * The array is modified in-place and unreferenced duplicate fences are released
91 * via dma_fence_put(). The function returns the new number of fences after
92 * deduplication.
93 *
94 * Return: Number of unique fences remaining in the array.
95 */
dma_fence_dedup_array(struct dma_fence ** fences,size_t num_fences)96 size_t dma_fence_dedup_array(struct dma_fence **fences, size_t num_fences)
97 {
98 size_t i, j;
99
100 if (!num_fences)
101 return 0;
102
103 sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL);
104
105 /*
106 * Only keep the most recent fence for each context.
107 */
108 j = 0;
109 for (i = 1; i < num_fences; i++) {
110 if (fences[i]->context == fences[j]->context)
111 dma_fence_put(fences[i]);
112 else
113 fences[++j] = fences[i];
114 }
115
116 return ++j;
117 }
118 EXPORT_SYMBOL_GPL(dma_fence_dedup_array);
119
120 /* Implementation for the dma_fence_merge() marco, don't use directly */
__dma_fence_unwrap_merge(size_t num_fences,struct dma_fence ** fences,struct dma_fence_unwrap * iter)121 struct dma_fence *__dma_fence_unwrap_merge(size_t num_fences,
122 struct dma_fence **fences,
123 struct dma_fence_unwrap *iter)
124 {
125 struct dma_fence *tmp, *unsignaled = NULL, **array;
126 struct dma_fence_array *result;
127 ktime_t timestamp;
128 size_t i, count;
129
130 count = 0;
131 timestamp = ns_to_ktime(0);
132 for (i = 0; i < num_fences; ++i) {
133 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
134 if (!dma_fence_is_signaled(tmp)) {
135 dma_fence_put(unsignaled);
136 unsignaled = dma_fence_get(tmp);
137 ++count;
138 } else {
139 ktime_t t = dma_fence_timestamp(tmp);
140
141 if (ktime_after(t, timestamp))
142 timestamp = t;
143 }
144 }
145 }
146
147 /*
148 * If we couldn't find a pending fence just return a private signaled
149 * fence with the timestamp of the last signaled one.
150 *
151 * Or if there was a single unsignaled fence left we can return it
152 * directly and early since that is a major path on many workloads.
153 */
154 if (count == 0)
155 return dma_fence_allocate_private_stub(timestamp);
156 else if (count == 1)
157 return unsignaled;
158
159 dma_fence_put(unsignaled);
160
161 array = kmalloc_objs(*array, count);
162 if (!array)
163 return NULL;
164
165 count = 0;
166 for (i = 0; i < num_fences; ++i) {
167 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
168 if (!dma_fence_is_signaled(tmp)) {
169 array[count++] = dma_fence_get(tmp);
170 } else {
171 ktime_t t = dma_fence_timestamp(tmp);
172
173 if (ktime_after(t, timestamp))
174 timestamp = t;
175 }
176 }
177 }
178
179 if (count == 0 || count == 1)
180 goto return_fastpath;
181
182 count = dma_fence_dedup_array(array, count);
183
184 if (count > 1) {
185 result = dma_fence_array_create(count, array,
186 dma_fence_context_alloc(1), 1);
187 if (!result) {
188 for (i = 0; i < count; i++)
189 dma_fence_put(array[i]);
190 tmp = NULL;
191 goto return_tmp;
192 }
193 return &result->base;
194 }
195
196 return_fastpath:
197 if (count == 0)
198 tmp = dma_fence_allocate_private_stub(timestamp);
199 else
200 tmp = array[0];
201
202 return_tmp:
203 kfree(array);
204 return tmp;
205 }
206 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);
207