1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2
3 #include <drm/drm_exec.h>
4 #include <drm/drm_gem.h>
5 #include <linux/dma-resv.h>
6
7 /**
8 * DOC: Overview
9 *
10 * This component mainly abstracts the retry loop necessary for locking
11 * multiple GEM objects while preparing hardware operations (e.g. command
12 * submissions, page table updates etc..).
13 *
14 * If a contention is detected while locking a GEM object the cleanup procedure
15 * unlocks all previously locked GEM objects and locks the contended one first
16 * before locking any further objects.
17 *
18 * After an object is locked fences slots can optionally be reserved on the
19 * dma_resv object inside the GEM object.
20 *
21 * A typical usage pattern should look like this::
22 *
23 * struct drm_gem_object *obj;
24 * struct drm_exec exec;
25 * unsigned long index;
26 * int ret;
27 *
28 * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
29 * drm_exec_until_all_locked(&exec) {
30 * ret = drm_exec_prepare_obj(&exec, boA, 1);
31 * drm_exec_retry_on_contention(&exec);
32 * if (ret)
33 * goto error;
34 *
35 * ret = drm_exec_prepare_obj(&exec, boB, 1);
36 * drm_exec_retry_on_contention(&exec);
37 * if (ret)
38 * goto error;
39 * }
40 *
41 * drm_exec_for_each_locked_object(&exec, index, obj) {
42 * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ);
43 * ...
44 * }
45 * drm_exec_fini(&exec);
46 *
47 * See struct dma_exec for more details.
48 */
49
50 /* Dummy value used to initially enter the retry loop */
51 #define DRM_EXEC_DUMMY ((void *)~0)
52
53 /* Unlock all objects and drop references */
drm_exec_unlock_all(struct drm_exec * exec)54 static void drm_exec_unlock_all(struct drm_exec *exec)
55 {
56 struct drm_gem_object *obj;
57 unsigned long index;
58
59 drm_exec_for_each_locked_object_reverse(exec, index, obj) {
60 dma_resv_unlock(obj->resv);
61 drm_gem_object_put(obj);
62 }
63
64 drm_gem_object_put(exec->prelocked);
65 exec->prelocked = NULL;
66 }
67
68 /**
69 * drm_exec_init - initialize a drm_exec object
70 * @exec: the drm_exec object to initialize
71 * @flags: controls locking behavior, see DRM_EXEC_* defines
72 * @nr: the initial # of objects
73 *
74 * Initialize the object and make sure that we can track locked objects.
75 *
76 * If nr is non-zero then it is used as the initial objects table size.
77 * In either case, the table will grow (be re-allocated) on demand.
78 */
drm_exec_init(struct drm_exec * exec,u32 flags,unsigned nr)79 void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr)
80 {
81 if (!nr)
82 nr = PAGE_SIZE / sizeof(void *);
83
84 exec->flags = flags;
85 exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL);
86
87 /* If allocation here fails, just delay that till the first use */
88 exec->max_objects = exec->objects ? nr : 0;
89 exec->num_objects = 0;
90 exec->contended = DRM_EXEC_DUMMY;
91 exec->prelocked = NULL;
92 }
93 EXPORT_SYMBOL(drm_exec_init);
94
95 /**
96 * drm_exec_fini - finalize a drm_exec object
97 * @exec: the drm_exec object to finalize
98 *
99 * Unlock all locked objects, drop the references to objects and free all memory
100 * used for tracking the state.
101 */
drm_exec_fini(struct drm_exec * exec)102 void drm_exec_fini(struct drm_exec *exec)
103 {
104 drm_exec_unlock_all(exec);
105 kvfree(exec->objects);
106 if (exec->contended != DRM_EXEC_DUMMY) {
107 drm_gem_object_put(exec->contended);
108 ww_acquire_fini(&exec->ticket);
109 }
110 }
111 EXPORT_SYMBOL(drm_exec_fini);
112
113 /**
114 * drm_exec_cleanup - cleanup when contention is detected
115 * @exec: the drm_exec object to cleanup
116 *
117 * Cleanup the current state and return true if we should stay inside the retry
118 * loop, false if there wasn't any contention detected and we can keep the
119 * objects locked.
120 */
drm_exec_cleanup(struct drm_exec * exec)121 bool drm_exec_cleanup(struct drm_exec *exec)
122 {
123 if (likely(!exec->contended)) {
124 ww_acquire_done(&exec->ticket);
125 return false;
126 }
127
128 if (likely(exec->contended == DRM_EXEC_DUMMY)) {
129 exec->contended = NULL;
130 ww_acquire_init(&exec->ticket, &reservation_ww_class);
131 return true;
132 }
133
134 drm_exec_unlock_all(exec);
135 exec->num_objects = 0;
136 return true;
137 }
138 EXPORT_SYMBOL(drm_exec_cleanup);
139
140 /* Track the locked object in the array */
drm_exec_obj_locked(struct drm_exec * exec,struct drm_gem_object * obj)141 static int drm_exec_obj_locked(struct drm_exec *exec,
142 struct drm_gem_object *obj)
143 {
144 if (unlikely(exec->num_objects == exec->max_objects)) {
145 size_t size = exec->max_objects * sizeof(void *);
146 void *tmp;
147
148 tmp = kvrealloc(exec->objects, size + PAGE_SIZE, GFP_KERNEL);
149 if (!tmp)
150 return -ENOMEM;
151
152 exec->objects = tmp;
153 exec->max_objects += PAGE_SIZE / sizeof(void *);
154 }
155 drm_gem_object_get(obj);
156 exec->objects[exec->num_objects++] = obj;
157
158 return 0;
159 }
160
161 /* Make sure the contended object is locked first */
drm_exec_lock_contended(struct drm_exec * exec)162 static int drm_exec_lock_contended(struct drm_exec *exec)
163 {
164 struct drm_gem_object *obj = exec->contended;
165 int ret;
166
167 if (likely(!obj))
168 return 0;
169
170 /* Always cleanup the contention so that error handling can kick in */
171 exec->contended = NULL;
172 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) {
173 ret = dma_resv_lock_slow_interruptible(obj->resv,
174 &exec->ticket);
175 if (unlikely(ret))
176 goto error_dropref;
177 } else {
178 dma_resv_lock_slow(obj->resv, &exec->ticket);
179 }
180
181 ret = drm_exec_obj_locked(exec, obj);
182 if (unlikely(ret))
183 goto error_unlock;
184
185 exec->prelocked = obj;
186 return 0;
187
188 error_unlock:
189 dma_resv_unlock(obj->resv);
190
191 error_dropref:
192 drm_gem_object_put(obj);
193 return ret;
194 }
195
196 /**
197 * drm_exec_lock_obj - lock a GEM object for use
198 * @exec: the drm_exec object with the state
199 * @obj: the GEM object to lock
200 *
201 * Lock a GEM object for use and grab a reference to it.
202 *
203 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
204 * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES
205 * flag), -ENOMEM when memory allocation failed and zero for success.
206 */
drm_exec_lock_obj(struct drm_exec * exec,struct drm_gem_object * obj)207 int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
208 {
209 int ret;
210
211 ret = drm_exec_lock_contended(exec);
212 if (unlikely(ret))
213 return ret;
214
215 if (exec->prelocked == obj) {
216 drm_gem_object_put(exec->prelocked);
217 exec->prelocked = NULL;
218 return 0;
219 }
220
221 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT)
222 ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket);
223 else
224 ret = dma_resv_lock(obj->resv, &exec->ticket);
225
226 if (unlikely(ret == -EDEADLK)) {
227 drm_gem_object_get(obj);
228 exec->contended = obj;
229 return -EDEADLK;
230 }
231
232 if (unlikely(ret == -EALREADY) &&
233 exec->flags & DRM_EXEC_IGNORE_DUPLICATES)
234 return 0;
235
236 if (unlikely(ret))
237 return ret;
238
239 ret = drm_exec_obj_locked(exec, obj);
240 if (ret)
241 goto error_unlock;
242
243 return 0;
244
245 error_unlock:
246 dma_resv_unlock(obj->resv);
247 return ret;
248 }
249 EXPORT_SYMBOL(drm_exec_lock_obj);
250
251 /**
252 * drm_exec_unlock_obj - unlock a GEM object in this exec context
253 * @exec: the drm_exec object with the state
254 * @obj: the GEM object to unlock
255 *
256 * Unlock the GEM object and remove it from the collection of locked objects.
257 * Should only be used to unlock the most recently locked objects. It's not time
258 * efficient to unlock objects locked long ago.
259 */
drm_exec_unlock_obj(struct drm_exec * exec,struct drm_gem_object * obj)260 void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
261 {
262 unsigned int i;
263
264 for (i = exec->num_objects; i--;) {
265 if (exec->objects[i] == obj) {
266 dma_resv_unlock(obj->resv);
267 for (++i; i < exec->num_objects; ++i)
268 exec->objects[i - 1] = exec->objects[i];
269 --exec->num_objects;
270 drm_gem_object_put(obj);
271 return;
272 }
273
274 }
275 }
276 EXPORT_SYMBOL(drm_exec_unlock_obj);
277
278 /**
279 * drm_exec_prepare_obj - prepare a GEM object for use
280 * @exec: the drm_exec object with the state
281 * @obj: the GEM object to prepare
282 * @num_fences: how many fences to reserve
283 *
284 * Prepare a GEM object for use by locking it and reserving fence slots.
285 *
286 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
287 * already locked, -ENOMEM when memory allocation failed and zero for success.
288 */
drm_exec_prepare_obj(struct drm_exec * exec,struct drm_gem_object * obj,unsigned int num_fences)289 int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
290 unsigned int num_fences)
291 {
292 int ret;
293
294 ret = drm_exec_lock_obj(exec, obj);
295 if (ret)
296 return ret;
297
298 ret = dma_resv_reserve_fences(obj->resv, num_fences);
299 if (ret) {
300 drm_exec_unlock_obj(exec, obj);
301 return ret;
302 }
303
304 return 0;
305 }
306 EXPORT_SYMBOL(drm_exec_prepare_obj);
307
308 /**
309 * drm_exec_prepare_array - helper to prepare an array of objects
310 * @exec: the drm_exec object with the state
311 * @objects: array of GEM object to prepare
312 * @num_objects: number of GEM objects in the array
313 * @num_fences: number of fences to reserve on each GEM object
314 *
315 * Prepares all GEM objects in an array, aborts on first error.
316 * Reserves @num_fences on each GEM object after locking it.
317 *
318 * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked,
319 * -ENOMEM when memory allocation failed and zero for success.
320 */
drm_exec_prepare_array(struct drm_exec * exec,struct drm_gem_object ** objects,unsigned int num_objects,unsigned int num_fences)321 int drm_exec_prepare_array(struct drm_exec *exec,
322 struct drm_gem_object **objects,
323 unsigned int num_objects,
324 unsigned int num_fences)
325 {
326 int ret;
327
328 for (unsigned int i = 0; i < num_objects; ++i) {
329 ret = drm_exec_prepare_obj(exec, objects[i], num_fences);
330 if (unlikely(ret))
331 return ret;
332 }
333
334 return 0;
335 }
336 EXPORT_SYMBOL(drm_exec_prepare_array);
337
338 MODULE_DESCRIPTION("DRM execution context");
339 MODULE_LICENSE("Dual MIT/GPL");
340