1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright (C) 2014-2018 Broadcom
4 * Copyright (C) 2023 Raspberry Pi
5 */
6
7 #include <drm/drm_print.h>
8 #include <drm/drm_syncobj.h>
9
10 #include "v3d_drv.h"
11 #include "v3d_regs.h"
12 #include "v3d_trace.h"
13
14 /* Takes the reservation lock on all the BOs being referenced, so that
15 * we can attach fences and update the reservations after pushing the job
16 * to the queue.
17 *
18 * We don't lock the RCL the tile alloc/state BOs, or overflow memory
19 * (all of which are on render->unref_list). They're entirely private
20 * to v3d, so we don't attach dma-buf fences to them.
21 */
22 static int
v3d_lock_bo_reservations(struct v3d_job * job,struct ww_acquire_ctx * acquire_ctx)23 v3d_lock_bo_reservations(struct v3d_job *job,
24 struct ww_acquire_ctx *acquire_ctx)
25 {
26 int i, ret;
27
28 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx);
29 if (ret)
30 return ret;
31
32 for (i = 0; i < job->bo_count; i++) {
33 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1);
34 if (ret)
35 goto fail;
36
37 ret = drm_sched_job_add_implicit_dependencies(&job->base,
38 job->bo[i], true);
39 if (ret)
40 goto fail;
41 }
42
43 return 0;
44
45 fail:
46 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
47 return ret;
48 }
49
50 /**
51 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
52 * referenced by the job.
53 * @dev: DRM device
54 * @file_priv: DRM file for this fd
55 * @job: V3D job being set up
56 * @bo_handles: GEM handles
57 * @bo_count: Number of GEM handles passed in
58 *
59 * The command validator needs to reference BOs by their index within
60 * the submitted job's BO list. This does the validation of the job's
61 * BO list and reference counting for the lifetime of the job.
62 *
63 * Note that this function doesn't need to unreference the BOs on
64 * failure, because that will happen at `v3d_job_free()`.
65 */
66 static int
v3d_lookup_bos(struct drm_device * dev,struct drm_file * file_priv,struct v3d_job * job,u64 bo_handles,u32 bo_count)67 v3d_lookup_bos(struct drm_device *dev,
68 struct drm_file *file_priv,
69 struct v3d_job *job,
70 u64 bo_handles,
71 u32 bo_count)
72 {
73 job->bo_count = bo_count;
74
75 if (!job->bo_count) {
76 /* See comment on bo_index for why we have to check
77 * this.
78 */
79 drm_warn(dev, "Rendering requires BOs\n");
80 return -EINVAL;
81 }
82
83 return drm_gem_objects_lookup(file_priv,
84 (void __user *)(uintptr_t)bo_handles,
85 job->bo_count, &job->bo);
86 }
87
88 static void
v3d_job_free(struct kref * ref)89 v3d_job_free(struct kref *ref)
90 {
91 struct v3d_job *job = container_of(ref, struct v3d_job, refcount);
92 int i;
93
94 if (job->bo) {
95 for (i = 0; i < job->bo_count; i++)
96 drm_gem_object_put(job->bo[i]);
97 kvfree(job->bo);
98 }
99
100 dma_fence_put(job->irq_fence);
101 dma_fence_put(job->done_fence);
102
103 if (job->perfmon)
104 v3d_perfmon_put(job->perfmon);
105
106 v3d_stats_put(job->client_stats);
107 v3d_stats_put(job->global_stats);
108
109 if (job->has_pm_ref)
110 v3d_pm_runtime_put(job->v3d);
111
112 kfree(job);
113 }
114
115 static void
v3d_render_job_free(struct kref * ref)116 v3d_render_job_free(struct kref *ref)
117 {
118 struct v3d_render_job *job = container_of(ref, struct v3d_render_job,
119 base.refcount);
120 struct v3d_bo *bo, *save;
121
122 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {
123 drm_gem_object_put(&bo->base.base);
124 }
125
126 v3d_job_free(ref);
127 }
128
129 static void
v3d_cpu_job_free(struct kref * ref)130 v3d_cpu_job_free(struct kref *ref)
131 {
132 struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job,
133 base.refcount);
134
135 v3d_timestamp_query_info_free(&job->timestamp_query,
136 job->timestamp_query.count);
137
138 v3d_performance_query_info_free(&job->performance_query,
139 job->performance_query.count);
140
141 if (job->indirect_csd.indirect)
142 drm_gem_object_put(job->indirect_csd.indirect);
143
144 v3d_job_free(ref);
145 }
146
v3d_job_cleanup(struct v3d_job * job)147 void v3d_job_cleanup(struct v3d_job *job)
148 {
149 if (!job)
150 return;
151
152 drm_sched_job_cleanup(&job->base);
153 v3d_job_put(job);
154 }
155
v3d_job_put(struct v3d_job * job)156 void v3d_job_put(struct v3d_job *job)
157 {
158 if (!job)
159 return;
160
161 kref_put(&job->refcount, job->free);
162 }
163
164 static int
v3d_job_allocate(struct v3d_dev * v3d,void ** container,size_t size)165 v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size)
166 {
167 *container = kcalloc(1, size, GFP_KERNEL);
168 if (!*container) {
169 drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n");
170 return -ENOMEM;
171 }
172
173 return 0;
174 }
175
176 static void
v3d_job_deallocate(void ** container)177 v3d_job_deallocate(void **container)
178 {
179 kfree(*container);
180 *container = NULL;
181 }
182
183 static int
v3d_job_init(struct v3d_dev * v3d,struct drm_file * file_priv,struct v3d_job * job,void (* free)(struct kref * ref),u32 in_sync,struct v3d_submit_ext * se,enum v3d_queue queue)184 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
185 struct v3d_job *job, void (*free)(struct kref *ref),
186 u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue)
187 {
188 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
189 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
190 int ret, i;
191
192 job->v3d = v3d;
193 job->free = free;
194 job->file_priv = v3d_priv;
195
196 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
197 1, v3d_priv, file_priv->client_id);
198 if (ret)
199 return ret;
200
201 if (has_multisync) {
202 if (se->in_sync_count && se->wait_stage == queue) {
203 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
204
205 for (i = 0; i < se->in_sync_count; i++) {
206 struct drm_v3d_sem in;
207
208 if (copy_from_user(&in, handle++, sizeof(in))) {
209 ret = -EFAULT;
210 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
211 goto fail_job_init;
212 }
213 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);
214
215 // TODO: Investigate why this was filtered out for the IOCTL.
216 if (ret && ret != -ENOENT)
217 goto fail_job_init;
218 }
219 }
220 } else {
221 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0);
222
223 // TODO: Investigate why this was filtered out for the IOCTL.
224 if (ret && ret != -ENOENT)
225 goto fail_job_init;
226 }
227
228 /* CPU jobs don't require hardware resources */
229 if (queue != V3D_CPU) {
230 ret = v3d_pm_runtime_get(v3d);
231 if (ret)
232 goto fail_job_init;
233 job->has_pm_ref = true;
234 }
235
236 kref_init(&job->refcount);
237
238 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]);
239 job->global_stats = v3d_stats_get(v3d->queue[queue].stats);
240
241 return 0;
242
243 fail_job_init:
244 drm_sched_job_cleanup(&job->base);
245 return ret;
246 }
247
248 static void
v3d_push_job(struct v3d_job * job)249 v3d_push_job(struct v3d_job *job)
250 {
251 drm_sched_job_arm(&job->base);
252
253 job->done_fence = dma_fence_get(&job->base.s_fence->finished);
254
255 /* put by scheduler job completion */
256 kref_get(&job->refcount);
257
258 drm_sched_entity_push_job(&job->base);
259 }
260
261 static void
v3d_attach_fences_and_unlock_reservation(struct drm_file * file_priv,struct v3d_job * job,struct ww_acquire_ctx * acquire_ctx,u32 out_sync,struct v3d_submit_ext * se,struct dma_fence * done_fence)262 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv,
263 struct v3d_job *job,
264 struct ww_acquire_ctx *acquire_ctx,
265 u32 out_sync,
266 struct v3d_submit_ext *se,
267 struct dma_fence *done_fence)
268 {
269 struct drm_syncobj *sync_out;
270 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
271 int i;
272
273 for (i = 0; i < job->bo_count; i++) {
274 /* XXX: Use shared fences for read-only objects. */
275 dma_resv_add_fence(job->bo[i]->resv, job->done_fence,
276 DMA_RESV_USAGE_WRITE);
277 }
278
279 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
280
281 /* Update the return sync object for the job */
282 /* If it only supports a single signal semaphore*/
283 if (!has_multisync) {
284 sync_out = drm_syncobj_find(file_priv, out_sync);
285 if (sync_out) {
286 drm_syncobj_replace_fence(sync_out, done_fence);
287 drm_syncobj_put(sync_out);
288 }
289 return;
290 }
291
292 /* If multiple semaphores extension is supported */
293 if (se->out_sync_count) {
294 for (i = 0; i < se->out_sync_count; i++) {
295 drm_syncobj_replace_fence(se->out_syncs[i].syncobj,
296 done_fence);
297 drm_syncobj_put(se->out_syncs[i].syncobj);
298 }
299 kvfree(se->out_syncs);
300 }
301 }
302
303 static int
v3d_setup_csd_jobs_and_bos(struct drm_file * file_priv,struct v3d_dev * v3d,struct drm_v3d_submit_csd * args,struct v3d_csd_job ** job,struct v3d_job ** clean_job,struct v3d_submit_ext * se,struct ww_acquire_ctx * acquire_ctx)304 v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv,
305 struct v3d_dev *v3d,
306 struct drm_v3d_submit_csd *args,
307 struct v3d_csd_job **job,
308 struct v3d_job **clean_job,
309 struct v3d_submit_ext *se,
310 struct ww_acquire_ctx *acquire_ctx)
311 {
312 int ret;
313
314 ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job));
315 if (ret)
316 return ret;
317
318 ret = v3d_job_init(v3d, file_priv, &(*job)->base,
319 v3d_job_free, args->in_sync, se, V3D_CSD);
320 if (ret) {
321 v3d_job_deallocate((void *)job);
322 return ret;
323 }
324
325 ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job));
326 if (ret)
327 return ret;
328
329 ret = v3d_job_init(v3d, file_priv, *clean_job,
330 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
331 if (ret) {
332 v3d_job_deallocate((void *)clean_job);
333 return ret;
334 }
335
336 (*job)->args = *args;
337
338 ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job,
339 args->bo_handles, args->bo_handle_count);
340 if (ret)
341 return ret;
342
343 return v3d_lock_bo_reservations(*clean_job, acquire_ctx);
344 }
345
346 static void
v3d_put_multisync_post_deps(struct v3d_submit_ext * se)347 v3d_put_multisync_post_deps(struct v3d_submit_ext *se)
348 {
349 unsigned int i;
350
351 if (!(se && se->out_sync_count))
352 return;
353
354 for (i = 0; i < se->out_sync_count; i++)
355 drm_syncobj_put(se->out_syncs[i].syncobj);
356 kvfree(se->out_syncs);
357 }
358
359 static int
v3d_get_multisync_post_deps(struct drm_file * file_priv,struct v3d_submit_ext * se,u32 count,u64 handles)360 v3d_get_multisync_post_deps(struct drm_file *file_priv,
361 struct v3d_submit_ext *se,
362 u32 count, u64 handles)
363 {
364 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
365 struct v3d_dev *v3d = v3d_priv->v3d;
366 struct drm_v3d_sem __user *post_deps;
367 int i, ret;
368
369 if (!count)
370 return 0;
371
372 se->out_syncs = (struct v3d_submit_outsync *)
373 kvmalloc_objs(struct v3d_submit_outsync, count);
374 if (!se->out_syncs)
375 return -ENOMEM;
376
377 post_deps = u64_to_user_ptr(handles);
378
379 for (i = 0; i < count; i++) {
380 struct drm_v3d_sem out;
381
382 if (copy_from_user(&out, post_deps++, sizeof(out))) {
383 ret = -EFAULT;
384 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n");
385 goto fail;
386 }
387
388 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv,
389 out.handle);
390 if (!se->out_syncs[i].syncobj) {
391 ret = -EINVAL;
392 goto fail;
393 }
394 }
395 se->out_sync_count = count;
396
397 return 0;
398
399 fail:
400 for (i--; i >= 0; i--)
401 drm_syncobj_put(se->out_syncs[i].syncobj);
402 kvfree(se->out_syncs);
403
404 return ret;
405 }
406
407 /* Get data for multiple binary semaphores synchronization. Parse syncobj
408 * to be signaled when job completes (out_sync).
409 */
410 static int
v3d_get_multisync_submit_deps(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_submit_ext * se)411 v3d_get_multisync_submit_deps(struct drm_file *file_priv,
412 struct drm_v3d_extension __user *ext,
413 struct v3d_submit_ext *se)
414 {
415 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
416 struct v3d_dev *v3d = v3d_priv->v3d;
417 struct drm_v3d_multi_sync multisync;
418 int ret;
419
420 if (se->in_sync_count || se->out_sync_count) {
421 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job.");
422 return -EINVAL;
423 }
424
425 if (copy_from_user(&multisync, ext, sizeof(multisync)))
426 return -EFAULT;
427
428 if (multisync.pad)
429 return -EINVAL;
430
431 if (!multisync.in_sync_count && !multisync.out_sync_count) {
432 drm_dbg(&v3d->drm, "Empty multisync extension\n");
433 return -EINVAL;
434 }
435
436 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count,
437 multisync.out_syncs);
438 if (ret)
439 return ret;
440
441 se->in_sync_count = multisync.in_sync_count;
442 se->in_syncs = multisync.in_syncs;
443 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC;
444 se->wait_stage = multisync.wait_stage;
445
446 return 0;
447 }
448
449 /* Returns false if the CPU job has an invalid configuration. */
450 static bool
v3d_validate_cpu_job(struct drm_file * file_priv,struct v3d_cpu_job * job)451 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job)
452 {
453 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
454 struct v3d_dev *v3d = v3d_priv->v3d;
455
456 if (!job) {
457 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n");
458 return false;
459 }
460
461 if (job->job_type) {
462 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n");
463 return false;
464 }
465
466 return true;
467 }
468
469 /* Get data for the indirect CSD job submission. */
470 static int
v3d_get_cpu_indirect_csd_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)471 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv,
472 struct drm_v3d_extension __user *ext,
473 struct v3d_cpu_job *job)
474 {
475 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
476 struct v3d_dev *v3d = v3d_priv->v3d;
477 struct drm_v3d_indirect_csd indirect_csd;
478 struct v3d_indirect_csd_info *info = &job->indirect_csd;
479
480 if (!v3d_validate_cpu_job(file_priv, job))
481 return -EINVAL;
482
483 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd)))
484 return -EFAULT;
485
486 if (!v3d_has_csd(v3d)) {
487 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n");
488 return -EINVAL;
489 }
490
491 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;
492 info->offset = indirect_csd.offset;
493 info->wg_size = indirect_csd.wg_size;
494 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets,
495 sizeof(indirect_csd.wg_uniform_offsets));
496
497 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect);
498 if (!info->indirect)
499 return -ENOENT;
500
501 return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit,
502 &info->job, &info->clean_job,
503 NULL, &info->acquire_ctx);
504 }
505
506 /* Get data for the query timestamp job submission. */
507 static int
v3d_get_cpu_timestamp_query_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)508 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv,
509 struct drm_v3d_extension __user *ext,
510 struct v3d_cpu_job *job)
511 {
512 u32 __user *offsets, *syncs;
513 struct drm_v3d_timestamp_query timestamp;
514 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
515 unsigned int i;
516 int err;
517
518 if (!v3d_validate_cpu_job(file_priv, job))
519 return -EINVAL;
520
521 if (copy_from_user(×tamp, ext, sizeof(timestamp)))
522 return -EFAULT;
523
524 if (timestamp.pad)
525 return -EINVAL;
526
527 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY;
528
529 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
530 timestamp.count);
531 if (!query_info->queries)
532 return -ENOMEM;
533
534 offsets = u64_to_user_ptr(timestamp.offsets);
535 syncs = u64_to_user_ptr(timestamp.syncs);
536
537 for (i = 0; i < timestamp.count; i++) {
538 u32 offset, sync;
539
540 if (get_user(offset, offsets++)) {
541 err = -EFAULT;
542 goto error;
543 }
544
545 query_info->queries[i].offset = offset;
546
547 if (get_user(sync, syncs++)) {
548 err = -EFAULT;
549 goto error;
550 }
551
552 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
553 sync);
554 if (!query_info->queries[i].syncobj) {
555 err = -ENOENT;
556 goto error;
557 }
558 }
559 query_info->count = timestamp.count;
560
561 return 0;
562
563 error:
564 v3d_timestamp_query_info_free(&job->timestamp_query, i);
565 return err;
566 }
567
568 static int
v3d_get_cpu_reset_timestamp_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)569 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv,
570 struct drm_v3d_extension __user *ext,
571 struct v3d_cpu_job *job)
572 {
573 u32 __user *syncs;
574 struct drm_v3d_reset_timestamp_query reset;
575 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
576 unsigned int i;
577 int err;
578
579 if (!v3d_validate_cpu_job(file_priv, job))
580 return -EINVAL;
581
582 if (copy_from_user(&reset, ext, sizeof(reset)))
583 return -EFAULT;
584
585 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY;
586
587 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
588 reset.count);
589 if (!query_info->queries)
590 return -ENOMEM;
591
592 syncs = u64_to_user_ptr(reset.syncs);
593
594 for (i = 0; i < reset.count; i++) {
595 u32 sync;
596
597 query_info->queries[i].offset = reset.offset + 8 * i;
598
599 if (get_user(sync, syncs++)) {
600 err = -EFAULT;
601 goto error;
602 }
603
604 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
605 sync);
606 if (!query_info->queries[i].syncobj) {
607 err = -ENOENT;
608 goto error;
609 }
610 }
611 query_info->count = reset.count;
612
613 return 0;
614
615 error:
616 v3d_timestamp_query_info_free(&job->timestamp_query, i);
617 return err;
618 }
619
620 /* Get data for the copy timestamp query results job submission. */
621 static int
v3d_get_cpu_copy_query_results_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)622 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv,
623 struct drm_v3d_extension __user *ext,
624 struct v3d_cpu_job *job)
625 {
626 u32 __user *offsets, *syncs;
627 struct drm_v3d_copy_timestamp_query copy;
628 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
629 unsigned int i;
630 int err;
631
632 if (!v3d_validate_cpu_job(file_priv, job))
633 return -EINVAL;
634
635 if (copy_from_user(©, ext, sizeof(copy)))
636 return -EFAULT;
637
638 if (copy.pad)
639 return -EINVAL;
640
641 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY;
642
643 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
644 copy.count);
645 if (!query_info->queries)
646 return -ENOMEM;
647
648 offsets = u64_to_user_ptr(copy.offsets);
649 syncs = u64_to_user_ptr(copy.syncs);
650
651 for (i = 0; i < copy.count; i++) {
652 u32 offset, sync;
653
654 if (get_user(offset, offsets++)) {
655 err = -EFAULT;
656 goto error;
657 }
658
659 query_info->queries[i].offset = offset;
660
661 if (get_user(sync, syncs++)) {
662 err = -EFAULT;
663 goto error;
664 }
665
666 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
667 sync);
668 if (!query_info->queries[i].syncobj) {
669 err = -ENOENT;
670 goto error;
671 }
672 }
673 query_info->count = copy.count;
674
675 job->copy.do_64bit = copy.do_64bit;
676 job->copy.do_partial = copy.do_partial;
677 job->copy.availability_bit = copy.availability_bit;
678 job->copy.offset = copy.offset;
679 job->copy.stride = copy.stride;
680
681 return 0;
682
683 error:
684 v3d_timestamp_query_info_free(&job->timestamp_query, i);
685 return err;
686 }
687
688 static int
v3d_copy_query_info(struct v3d_performance_query_info * query_info,unsigned int count,unsigned int nperfmons,u32 __user * syncs,u64 __user * kperfmon_ids,struct drm_file * file_priv)689 v3d_copy_query_info(struct v3d_performance_query_info *query_info,
690 unsigned int count,
691 unsigned int nperfmons,
692 u32 __user *syncs,
693 u64 __user *kperfmon_ids,
694 struct drm_file *file_priv)
695 {
696 unsigned int i, j;
697 int err;
698
699 for (i = 0; i < count; i++) {
700 struct v3d_performance_query *query = &query_info->queries[i];
701 u32 __user *ids_pointer;
702 u32 sync, id;
703 u64 ids;
704
705 if (get_user(sync, syncs++)) {
706 err = -EFAULT;
707 goto error;
708 }
709
710 if (get_user(ids, kperfmon_ids++)) {
711 err = -EFAULT;
712 goto error;
713 }
714
715 query->kperfmon_ids =
716 kvmalloc_array(nperfmons,
717 sizeof(struct v3d_performance_query *),
718 GFP_KERNEL);
719 if (!query->kperfmon_ids) {
720 err = -ENOMEM;
721 goto error;
722 }
723
724 ids_pointer = u64_to_user_ptr(ids);
725
726 for (j = 0; j < nperfmons; j++) {
727 if (get_user(id, ids_pointer++)) {
728 kvfree(query->kperfmon_ids);
729 err = -EFAULT;
730 goto error;
731 }
732
733 query->kperfmon_ids[j] = id;
734 }
735
736 query->syncobj = drm_syncobj_find(file_priv, sync);
737 if (!query->syncobj) {
738 kvfree(query->kperfmon_ids);
739 err = -ENOENT;
740 goto error;
741 }
742 }
743
744 return 0;
745
746 error:
747 v3d_performance_query_info_free(query_info, i);
748 return err;
749 }
750
751 static int
v3d_get_cpu_reset_performance_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)752 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv,
753 struct drm_v3d_extension __user *ext,
754 struct v3d_cpu_job *job)
755 {
756 struct v3d_performance_query_info *query_info = &job->performance_query;
757 struct drm_v3d_reset_performance_query reset;
758 int err;
759
760 if (!v3d_validate_cpu_job(file_priv, job))
761 return -EINVAL;
762
763 if (copy_from_user(&reset, ext, sizeof(reset)))
764 return -EFAULT;
765
766 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY;
767
768 query_info->queries =
769 kvmalloc_objs(struct v3d_performance_query, reset.count);
770 if (!query_info->queries)
771 return -ENOMEM;
772
773 err = v3d_copy_query_info(query_info,
774 reset.count,
775 reset.nperfmons,
776 u64_to_user_ptr(reset.syncs),
777 u64_to_user_ptr(reset.kperfmon_ids),
778 file_priv);
779 if (err)
780 return err;
781
782 query_info->count = reset.count;
783 query_info->nperfmons = reset.nperfmons;
784
785 return 0;
786 }
787
788 static int
v3d_get_cpu_copy_performance_query_params(struct drm_file * file_priv,struct drm_v3d_extension __user * ext,struct v3d_cpu_job * job)789 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv,
790 struct drm_v3d_extension __user *ext,
791 struct v3d_cpu_job *job)
792 {
793 struct v3d_performance_query_info *query_info = &job->performance_query;
794 struct drm_v3d_copy_performance_query copy;
795 int err;
796
797 if (!v3d_validate_cpu_job(file_priv, job))
798 return -EINVAL;
799
800 if (copy_from_user(©, ext, sizeof(copy)))
801 return -EFAULT;
802
803 if (copy.pad)
804 return -EINVAL;
805
806 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY;
807
808 query_info->queries =
809 kvmalloc_objs(struct v3d_performance_query, copy.count);
810 if (!query_info->queries)
811 return -ENOMEM;
812
813 err = v3d_copy_query_info(query_info,
814 copy.count,
815 copy.nperfmons,
816 u64_to_user_ptr(copy.syncs),
817 u64_to_user_ptr(copy.kperfmon_ids),
818 file_priv);
819 if (err)
820 return err;
821
822 query_info->count = copy.count;
823 query_info->nperfmons = copy.nperfmons;
824 query_info->ncounters = copy.ncounters;
825
826 job->copy.do_64bit = copy.do_64bit;
827 job->copy.do_partial = copy.do_partial;
828 job->copy.availability_bit = copy.availability_bit;
829 job->copy.offset = copy.offset;
830 job->copy.stride = copy.stride;
831
832 return 0;
833 }
834
835 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data
836 * according to the extension id (name).
837 */
838 static int
v3d_get_extensions(struct drm_file * file_priv,u64 ext_handles,struct v3d_submit_ext * se,struct v3d_cpu_job * job)839 v3d_get_extensions(struct drm_file *file_priv,
840 u64 ext_handles,
841 struct v3d_submit_ext *se,
842 struct v3d_cpu_job *job)
843 {
844 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
845 struct v3d_dev *v3d = v3d_priv->v3d;
846 struct drm_v3d_extension __user *user_ext;
847 int ret;
848
849 user_ext = u64_to_user_ptr(ext_handles);
850 while (user_ext) {
851 struct drm_v3d_extension ext;
852
853 if (copy_from_user(&ext, user_ext, sizeof(ext))) {
854 drm_dbg(&v3d->drm, "Failed to copy submit extension\n");
855 return -EFAULT;
856 }
857
858 switch (ext.id) {
859 case DRM_V3D_EXT_ID_MULTI_SYNC:
860 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se);
861 break;
862 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD:
863 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job);
864 break;
865 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY:
866 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job);
867 break;
868 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY:
869 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job);
870 break;
871 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY:
872 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job);
873 break;
874 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY:
875 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job);
876 break;
877 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY:
878 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job);
879 break;
880 default:
881 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id);
882 return -EINVAL;
883 }
884
885 if (ret)
886 return ret;
887
888 user_ext = u64_to_user_ptr(ext.next);
889 }
890
891 return 0;
892 }
893
894 /**
895 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
896 * @dev: DRM device
897 * @data: ioctl argument
898 * @file_priv: DRM file for this fd
899 *
900 * This is the main entrypoint for userspace to submit a 3D frame to
901 * the GPU. Userspace provides the binner command list (if
902 * applicable), and the kernel sets up the render command list to draw
903 * to the framebuffer described in the ioctl, using the command lists
904 * that the 3D engine's binner will produce.
905 */
906 int
v3d_submit_cl_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)907 v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
908 struct drm_file *file_priv)
909 {
910 struct v3d_dev *v3d = to_v3d_dev(dev);
911 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
912 struct drm_v3d_submit_cl *args = data;
913 struct v3d_submit_ext se = {0};
914 struct v3d_bin_job *bin = NULL;
915 struct v3d_render_job *render = NULL;
916 struct v3d_job *clean_job = NULL;
917 struct v3d_job *last_job;
918 struct ww_acquire_ctx acquire_ctx;
919 int ret = 0;
920
921 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end);
922
923 if (args->pad)
924 return -EINVAL;
925
926 if (args->flags &&
927 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE |
928 DRM_V3D_SUBMIT_EXTENSION)) {
929 drm_dbg(dev, "invalid flags: %d\n", args->flags);
930 return -EINVAL;
931 }
932
933 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
934 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
935 if (ret) {
936 drm_dbg(dev, "Failed to get extensions.\n");
937 return ret;
938 }
939 }
940
941 ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render));
942 if (ret)
943 return ret;
944
945 ret = v3d_job_init(v3d, file_priv, &render->base,
946 v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER);
947 if (ret) {
948 v3d_job_deallocate((void *)&render);
949 goto fail;
950 }
951
952 render->start = args->rcl_start;
953 render->end = args->rcl_end;
954 INIT_LIST_HEAD(&render->unref_list);
955
956 if (args->bcl_start != args->bcl_end) {
957 ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin));
958 if (ret)
959 goto fail;
960
961 ret = v3d_job_init(v3d, file_priv, &bin->base,
962 v3d_job_free, args->in_sync_bcl, &se, V3D_BIN);
963 if (ret) {
964 v3d_job_deallocate((void *)&bin);
965 goto fail;
966 }
967
968 bin->start = args->bcl_start;
969 bin->end = args->bcl_end;
970 bin->qma = args->qma;
971 bin->qms = args->qms;
972 bin->qts = args->qts;
973 bin->render = render;
974 }
975
976 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
977 ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job));
978 if (ret)
979 goto fail;
980
981 ret = v3d_job_init(v3d, file_priv, clean_job,
982 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
983 if (ret) {
984 v3d_job_deallocate((void *)&clean_job);
985 goto fail;
986 }
987
988 last_job = clean_job;
989 } else {
990 last_job = &render->base;
991 }
992
993 ret = v3d_lookup_bos(dev, file_priv, last_job,
994 args->bo_handles, args->bo_handle_count);
995 if (ret)
996 goto fail;
997
998 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx);
999 if (ret)
1000 goto fail;
1001
1002 if (args->perfmon_id) {
1003 if (v3d->global_perfmon) {
1004 ret = -EAGAIN;
1005 goto fail_perfmon;
1006 }
1007
1008 render->base.perfmon = v3d_perfmon_find(v3d_priv,
1009 args->perfmon_id);
1010
1011 if (!render->base.perfmon) {
1012 ret = -ENOENT;
1013 goto fail_perfmon;
1014 }
1015 }
1016
1017 mutex_lock(&v3d->sched_lock);
1018 if (bin) {
1019 bin->base.perfmon = render->base.perfmon;
1020 v3d_perfmon_get(bin->base.perfmon);
1021 v3d_push_job(&bin->base);
1022
1023 ret = drm_sched_job_add_dependency(&render->base.base,
1024 dma_fence_get(bin->base.done_fence));
1025 if (ret)
1026 goto fail_unreserve;
1027 }
1028
1029 v3d_push_job(&render->base);
1030
1031 if (clean_job) {
1032 struct dma_fence *render_fence =
1033 dma_fence_get(render->base.done_fence);
1034 ret = drm_sched_job_add_dependency(&clean_job->base,
1035 render_fence);
1036 if (ret)
1037 goto fail_unreserve;
1038 clean_job->perfmon = render->base.perfmon;
1039 v3d_perfmon_get(clean_job->perfmon);
1040 v3d_push_job(clean_job);
1041 }
1042
1043 mutex_unlock(&v3d->sched_lock);
1044
1045 v3d_attach_fences_and_unlock_reservation(file_priv,
1046 last_job,
1047 &acquire_ctx,
1048 args->out_sync,
1049 &se,
1050 last_job->done_fence);
1051
1052 v3d_job_put(&bin->base);
1053 v3d_job_put(&render->base);
1054 v3d_job_put(clean_job);
1055
1056 return 0;
1057
1058 fail_unreserve:
1059 mutex_unlock(&v3d->sched_lock);
1060 fail_perfmon:
1061 drm_gem_unlock_reservations(last_job->bo,
1062 last_job->bo_count, &acquire_ctx);
1063 fail:
1064 v3d_job_cleanup((void *)bin);
1065 v3d_job_cleanup((void *)render);
1066 v3d_job_cleanup(clean_job);
1067 v3d_put_multisync_post_deps(&se);
1068
1069 return ret;
1070 }
1071
1072 /**
1073 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
1074 * @dev: DRM device
1075 * @data: ioctl argument
1076 * @file_priv: DRM file for this fd
1077 *
1078 * Userspace provides the register setup for the TFU, which we don't
1079 * need to validate since the TFU is behind the MMU.
1080 */
1081 int
v3d_submit_tfu_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1082 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
1083 struct drm_file *file_priv)
1084 {
1085 struct v3d_dev *v3d = to_v3d_dev(dev);
1086 struct drm_v3d_submit_tfu *args = data;
1087 struct v3d_submit_ext se = {0};
1088 struct v3d_tfu_job *job = NULL;
1089 struct ww_acquire_ctx acquire_ctx;
1090 int ret = 0;
1091
1092 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia);
1093
1094 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1095 drm_dbg(dev, "invalid flags: %d\n", args->flags);
1096 return -EINVAL;
1097 }
1098
1099 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1100 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1101 if (ret) {
1102 drm_dbg(dev, "Failed to get extensions.\n");
1103 return ret;
1104 }
1105 }
1106
1107 ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job));
1108 if (ret)
1109 return ret;
1110
1111 ret = v3d_job_init(v3d, file_priv, &job->base,
1112 v3d_job_free, args->in_sync, &se, V3D_TFU);
1113 if (ret) {
1114 v3d_job_deallocate((void *)&job);
1115 goto fail;
1116 }
1117
1118 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles));
1119 if (!job->base.bo) {
1120 ret = -ENOMEM;
1121 goto fail;
1122 }
1123
1124 job->args = *args;
1125
1126 for (job->base.bo_count = 0;
1127 job->base.bo_count < ARRAY_SIZE(args->bo_handles);
1128 job->base.bo_count++) {
1129 struct drm_gem_object *bo;
1130
1131 if (!args->bo_handles[job->base.bo_count])
1132 break;
1133
1134 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]);
1135 if (!bo) {
1136 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n",
1137 job->base.bo_count,
1138 args->bo_handles[job->base.bo_count]);
1139 ret = -ENOENT;
1140 goto fail;
1141 }
1142 job->base.bo[job->base.bo_count] = bo;
1143 }
1144
1145 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx);
1146 if (ret)
1147 goto fail;
1148
1149 mutex_lock(&v3d->sched_lock);
1150 v3d_push_job(&job->base);
1151 mutex_unlock(&v3d->sched_lock);
1152
1153 v3d_attach_fences_and_unlock_reservation(file_priv,
1154 &job->base, &acquire_ctx,
1155 args->out_sync,
1156 &se,
1157 job->base.done_fence);
1158
1159 v3d_job_put(&job->base);
1160
1161 return 0;
1162
1163 fail:
1164 v3d_job_cleanup((void *)job);
1165 v3d_put_multisync_post_deps(&se);
1166
1167 return ret;
1168 }
1169
1170 /**
1171 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D.
1172 * @dev: DRM device
1173 * @data: ioctl argument
1174 * @file_priv: DRM file for this fd
1175 *
1176 * Userspace provides the register setup for the CSD, which we don't
1177 * need to validate since the CSD is behind the MMU.
1178 */
1179 int
v3d_submit_csd_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1180 v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
1181 struct drm_file *file_priv)
1182 {
1183 struct v3d_dev *v3d = to_v3d_dev(dev);
1184 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
1185 struct drm_v3d_submit_csd *args = data;
1186 struct v3d_submit_ext se = {0};
1187 struct v3d_csd_job *job = NULL;
1188 struct v3d_job *clean_job = NULL;
1189 struct ww_acquire_ctx acquire_ctx;
1190 int ret;
1191
1192 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]);
1193
1194 if (args->pad)
1195 return -EINVAL;
1196
1197 if (!v3d_has_csd(v3d)) {
1198 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n");
1199 return -EINVAL;
1200 }
1201
1202 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1203 drm_dbg(dev, "invalid flags: %d\n", args->flags);
1204 return -EINVAL;
1205 }
1206
1207 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1208 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1209 if (ret) {
1210 drm_dbg(dev, "Failed to get extensions.\n");
1211 return ret;
1212 }
1213 }
1214
1215 ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args,
1216 &job, &clean_job, &se,
1217 &acquire_ctx);
1218 if (ret)
1219 goto fail;
1220
1221 if (args->perfmon_id) {
1222 if (v3d->global_perfmon) {
1223 ret = -EAGAIN;
1224 goto fail_perfmon;
1225 }
1226
1227 job->base.perfmon = v3d_perfmon_find(v3d_priv,
1228 args->perfmon_id);
1229 if (!job->base.perfmon) {
1230 ret = -ENOENT;
1231 goto fail_perfmon;
1232 }
1233 }
1234
1235 mutex_lock(&v3d->sched_lock);
1236 v3d_push_job(&job->base);
1237
1238 ret = drm_sched_job_add_dependency(&clean_job->base,
1239 dma_fence_get(job->base.done_fence));
1240 if (ret)
1241 goto fail_unreserve;
1242
1243 v3d_push_job(clean_job);
1244 mutex_unlock(&v3d->sched_lock);
1245
1246 v3d_attach_fences_and_unlock_reservation(file_priv,
1247 clean_job,
1248 &acquire_ctx,
1249 args->out_sync,
1250 &se,
1251 clean_job->done_fence);
1252
1253 v3d_job_put(&job->base);
1254 v3d_job_put(clean_job);
1255
1256 return 0;
1257
1258 fail_unreserve:
1259 mutex_unlock(&v3d->sched_lock);
1260 fail_perfmon:
1261 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
1262 &acquire_ctx);
1263 fail:
1264 v3d_job_cleanup((void *)job);
1265 v3d_job_cleanup(clean_job);
1266 v3d_put_multisync_post_deps(&se);
1267
1268 return ret;
1269 }
1270
1271 static const unsigned int cpu_job_bo_handle_count[] = {
1272 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1,
1273 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1,
1274 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1,
1275 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2,
1276 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0,
1277 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1,
1278 };
1279
1280 /**
1281 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
1282 * @dev: DRM device
1283 * @data: ioctl argument
1284 * @file_priv: DRM file for this fd
1285 *
1286 * Userspace specifies the CPU job type and data required to perform its
1287 * operations through the drm_v3d_extension struct.
1288 */
1289 int
v3d_submit_cpu_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1290 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
1291 struct drm_file *file_priv)
1292 {
1293 struct v3d_dev *v3d = to_v3d_dev(dev);
1294 struct drm_v3d_submit_cpu *args = data;
1295 struct v3d_submit_ext se = {0};
1296 struct v3d_submit_ext *out_se = NULL;
1297 struct v3d_cpu_job *cpu_job = NULL;
1298 struct v3d_csd_job *csd_job = NULL;
1299 struct v3d_job *clean_job = NULL;
1300 struct ww_acquire_ctx acquire_ctx;
1301 int ret;
1302
1303 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1304 drm_dbg(dev, "Invalid flags: %d\n", args->flags);
1305 return -EINVAL;
1306 }
1307
1308 ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job));
1309 if (ret)
1310 return ret;
1311
1312 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1313 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job);
1314 if (ret) {
1315 drm_dbg(dev, "Failed to get extensions.\n");
1316 goto fail;
1317 }
1318 }
1319
1320 /* Every CPU job must have a CPU job user extension */
1321 if (!cpu_job->job_type) {
1322 drm_dbg(dev, "CPU job must have a CPU job user extension.\n");
1323 ret = -EINVAL;
1324 goto fail;
1325 }
1326
1327 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) {
1328 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n");
1329 ret = -EINVAL;
1330 goto fail;
1331 }
1332
1333 trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type);
1334
1335 ret = v3d_job_init(v3d, file_priv, &cpu_job->base,
1336 v3d_cpu_job_free, 0, &se, V3D_CPU);
1337 if (ret) {
1338 v3d_job_deallocate((void *)&cpu_job);
1339 goto fail;
1340 }
1341
1342 clean_job = cpu_job->indirect_csd.clean_job;
1343 csd_job = cpu_job->indirect_csd.job;
1344
1345 if (args->bo_handle_count) {
1346 ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base,
1347 args->bo_handles, args->bo_handle_count);
1348 if (ret)
1349 goto fail;
1350
1351 ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx);
1352 if (ret)
1353 goto fail;
1354 }
1355
1356 mutex_lock(&v3d->sched_lock);
1357 v3d_push_job(&cpu_job->base);
1358
1359 switch (cpu_job->job_type) {
1360 case V3D_CPU_JOB_TYPE_INDIRECT_CSD:
1361 ret = drm_sched_job_add_dependency(&csd_job->base.base,
1362 dma_fence_get(cpu_job->base.done_fence));
1363 if (ret)
1364 goto fail_unreserve;
1365
1366 v3d_push_job(&csd_job->base);
1367
1368 ret = drm_sched_job_add_dependency(&clean_job->base,
1369 dma_fence_get(csd_job->base.done_fence));
1370 if (ret)
1371 goto fail_unreserve;
1372
1373 v3d_push_job(clean_job);
1374
1375 break;
1376 default:
1377 break;
1378 }
1379 mutex_unlock(&v3d->sched_lock);
1380
1381 out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se;
1382
1383 v3d_attach_fences_and_unlock_reservation(file_priv,
1384 &cpu_job->base,
1385 &acquire_ctx, 0,
1386 out_se, cpu_job->base.done_fence);
1387
1388 switch (cpu_job->job_type) {
1389 case V3D_CPU_JOB_TYPE_INDIRECT_CSD:
1390 v3d_attach_fences_and_unlock_reservation(file_priv,
1391 clean_job,
1392 &cpu_job->indirect_csd.acquire_ctx,
1393 0, &se, clean_job->done_fence);
1394 break;
1395 default:
1396 break;
1397 }
1398
1399 v3d_job_put(&cpu_job->base);
1400 v3d_job_put(&csd_job->base);
1401 v3d_job_put(clean_job);
1402
1403 return 0;
1404
1405 fail_unreserve:
1406 mutex_unlock(&v3d->sched_lock);
1407
1408 drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count,
1409 &acquire_ctx);
1410
1411 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
1412 &cpu_job->indirect_csd.acquire_ctx);
1413
1414 fail:
1415 v3d_job_cleanup((void *)cpu_job);
1416 v3d_job_cleanup((void *)csd_job);
1417 v3d_job_cleanup(clean_job);
1418 v3d_put_multisync_post_deps(&se);
1419
1420 return ret;
1421 }
1422