xref: /linux/drivers/gpu/drm/v3d/v3d_submit.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (C) 2014-2018 Broadcom
4  * Copyright (C) 2023 Raspberry Pi
5  */
6 
7 #include <linux/dma-fence-unwrap.h>
8 #include <linux/overflow.h>
9 
10 #include <drm/drm_print.h>
11 #include <drm/drm_syncobj.h>
12 
13 #include "v3d_drv.h"
14 #include "v3d_regs.h"
15 #include "v3d_trace.h"
16 
17 /* Takes the reservation lock on all the BOs being referenced, so that
18  * we can attach fences and update the reservations after pushing the job
19  * to the queue.
20  *
21  * We don't lock the RCL the tile alloc/state BOs, or overflow memory
22  * (all of which are on render->unref_list). They're entirely private
23  * to v3d, so we don't attach dma-buf fences to them.
24  */
25 static int
26 v3d_submit_lock_reservations(struct v3d_submit *submit)
27 {
28 	int i, j, ret;
29 
30 	drm_exec_init(&submit->exec,
31 		      DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0);
32 	drm_exec_until_all_locked(&submit->exec) {
33 		for (i = 0; i < submit->job_count; i++) {
34 			struct v3d_job *job = submit->jobs[i];
35 
36 			ret = drm_exec_prepare_array(&submit->exec, job->bo,
37 						     job->bo_count, 1);
38 			if (ret)
39 				break;
40 		}
41 		drm_exec_retry_on_contention(&submit->exec);
42 		if (ret)
43 			goto fail;
44 	}
45 
46 	for (i = 0; i < submit->job_count; i++) {
47 		struct v3d_job *job = submit->jobs[i];
48 
49 		if (!job->has_implicit_dep)
50 			continue;
51 
52 		for (j = 0; j < job->bo_count; j++) {
53 			ret = drm_sched_job_add_implicit_dependencies(&job->base,
54 								      job->bo[j],
55 								      true);
56 			if (ret)
57 				goto fail;
58 		}
59 	}
60 
61 	return 0;
62 
63 fail:
64 	drm_exec_fini(&submit->exec);
65 	return ret;
66 }
67 
68 static void
69 v3d_submit_unlock_reservations(struct v3d_submit *submit)
70 {
71 	drm_exec_fini(&submit->exec);
72 }
73 
74 /**
75  * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
76  * referenced by the job.
77  * @file_priv: DRM file for this fd
78  * @job: V3D job being set up
79  * @bo_handles: GEM handles
80  * @bo_count: Number of GEM handles passed in
81  *
82  * The command validator needs to reference BOs by their index within
83  * the submitted job's BO list. This does the validation of the job's
84  * BO list and reference counting for the lifetime of the job.
85  *
86  * Note that this function doesn't need to unreference the BOs on
87  * failure, because that will happen at `v3d_job_free()`.
88  */
89 static int
90 v3d_lookup_bos(struct drm_file *file_priv, struct v3d_job *job,
91 	       u64 bo_handles, u32 bo_count)
92 {
93 	if (!bo_count) {
94 		drm_warn(&job->v3d->drm, "Rendering requires BOs\n");
95 		return -EINVAL;
96 	}
97 
98 	job->bo_count = bo_count;
99 
100 	return drm_gem_objects_lookup(file_priv,
101 				      (void __user *)(uintptr_t)bo_handles,
102 				      job->bo_count, &job->bo);
103 }
104 
105 /**
106  * v3d_job_reference_bos() - Share another job's BOs with @dst
107  * @dst: job that acquires references to the BOs
108  * @src: job whose already-resolved BO list is shared
109  *
110  * For submissions with multiple jobs that use the same BOs, a trailing job
111  * shouldn't look the handles up again, as it could cause inconsistencies.
112  * Instead, it should reference the previous job's BOs.
113  */
114 static int
115 v3d_job_reference_bos(struct v3d_job *dst, struct v3d_job *src)
116 {
117 	dst->bo = kvmalloc_objs(*dst->bo, src->bo_count);
118 	if (!dst->bo)
119 		return -ENOMEM;
120 
121 	dst->bo_count = src->bo_count;
122 	for (int i = 0; i < dst->bo_count; i++) {
123 		dst->bo[i] = src->bo[i];
124 		drm_gem_object_get(dst->bo[i]);
125 	}
126 
127 	return 0;
128 }
129 
130 static void
131 v3d_job_free(struct kref *ref)
132 {
133 	struct v3d_job *job = container_of(ref, struct v3d_job, refcount);
134 	int i;
135 
136 	if (job->bo) {
137 		for (i = 0; i < job->bo_count; i++)
138 			drm_gem_object_put(job->bo[i]);
139 		kvfree(job->bo);
140 	}
141 
142 	dma_fence_put(job->irq_fence);
143 	dma_fence_put(job->done_fence);
144 
145 	if (job->perfmon)
146 		v3d_perfmon_put(job->perfmon);
147 
148 	v3d_stats_put(job->client_stats);
149 	v3d_stats_put(job->global_stats);
150 
151 	if (job->has_pm_ref)
152 		v3d_pm_runtime_put(job->v3d);
153 
154 	kfree(job);
155 }
156 
157 static void
158 v3d_render_job_free(struct kref *ref)
159 {
160 	struct v3d_render_job *job = container_of(ref, struct v3d_render_job,
161 						  base.refcount);
162 	struct v3d_bo *bo, *save;
163 
164 	list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {
165 		drm_gem_object_put(&bo->base.base);
166 	}
167 
168 	v3d_job_free(ref);
169 }
170 
171 static void
172 v3d_cpu_job_free(struct kref *ref)
173 {
174 	struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job,
175 					       base.refcount);
176 
177 	v3d_timestamp_query_info_free(&job->timestamp_query,
178 				      job->timestamp_query.count);
179 
180 	v3d_performance_query_info_free(&job->performance_query,
181 					job->performance_query.count);
182 
183 	if (job->indirect_csd.indirect)
184 		drm_gem_object_put(job->indirect_csd.indirect);
185 
186 	v3d_job_free(ref);
187 }
188 
189 void v3d_job_cleanup(struct v3d_job *job)
190 {
191 	if (!job)
192 		return;
193 
194 	drm_sched_job_cleanup(&job->base);
195 	v3d_job_put(job);
196 }
197 
198 void v3d_job_put(struct v3d_job *job)
199 {
200 	if (!job)
201 		return;
202 
203 	kref_put(&job->refcount, job->free);
204 }
205 
206 static int
207 v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv,
208 		     u32 in_sync, struct v3d_submit_ext *se)
209 {
210 	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
211 	struct v3d_dev *v3d = job->v3d;
212 	int ret = 0;
213 
214 	if (!has_multisync) {
215 		/* Ignore syncobj if its handle is zero */
216 		if (in_sync)
217 			ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
218 								   in_sync, 0);
219 		return ret;
220 	}
221 
222 	if (se->in_sync_count && se->wait_stage == job->queue) {
223 		struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
224 
225 		for (int i = 0; i < se->in_sync_count; i++) {
226 			struct drm_v3d_sem in;
227 
228 			if (copy_from_user(&in, handle++, sizeof(in))) {
229 				drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
230 				return -EFAULT;
231 			}
232 
233 			/* Ignore syncobj if its handle is zero */
234 			if (in.handle) {
235 				ret = drm_sched_job_add_syncobj_dependency(&job->base,
236 									   file_priv, in.handle, 0);
237 				if (ret)
238 					return ret;
239 			}
240 		}
241 	}
242 
243 	return 0;
244 }
245 
246 static const struct {
247 	size_t size;
248 	void (*free)(struct kref *ref);
249 	bool has_implicit_dep;
250 } v3d_job_types[] = {
251 	[V3D_BIN]		= { sizeof(struct v3d_bin_job), v3d_job_free, false },
252 	[V3D_RENDER]		= { sizeof(struct v3d_render_job), v3d_render_job_free, true },
253 	[V3D_TFU]		= { sizeof(struct v3d_tfu_job), v3d_job_free, true },
254 	[V3D_CSD]		= { sizeof(struct v3d_csd_job), v3d_job_free, true },
255 	[V3D_CACHE_CLEAN]	= { sizeof(struct v3d_job), v3d_job_free, false },
256 	[V3D_CPU]		= { sizeof(struct v3d_cpu_job), v3d_cpu_job_free, true },
257 };
258 
259 static struct v3d_job *
260 v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue)
261 {
262 	struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
263 	struct v3d_dev *v3d = submit->v3d;
264 	struct v3d_job *job;
265 	int ret;
266 
267 	if (queue >= V3D_MAX_QUEUES)
268 		return ERR_PTR(-EINVAL);
269 
270 	job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL);
271 	if (!job)
272 		return ERR_PTR(-ENOMEM);
273 
274 	job->v3d = v3d;
275 	job->queue = queue;
276 	job->file_priv = v3d_priv;
277 	job->free = v3d_job_types[queue].free;
278 	job->has_implicit_dep = v3d_job_types[queue].has_implicit_dep;
279 
280 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
281 				 1, v3d_priv, submit->file_priv->client_id);
282 	if (ret)
283 		goto fail_free;
284 
285 	/* CPU jobs don't require hardware resources */
286 	if (queue != V3D_CPU) {
287 		ret = v3d_pm_runtime_get(v3d);
288 		if (ret)
289 			goto fail_sched_job;
290 		job->has_pm_ref = true;
291 	}
292 
293 	kref_init(&job->refcount);
294 
295 	job->client_stats = v3d_stats_get(v3d_priv->stats[queue]);
296 	job->global_stats = v3d_stats_get(v3d->queue[queue].stats);
297 
298 	submit->jobs[submit->job_count++] = job;
299 
300 	return job;
301 
302 fail_sched_job:
303 	drm_sched_job_cleanup(&job->base);
304 fail_free:
305 	kfree(job);
306 	return ERR_PTR(ret);
307 }
308 
309 static void
310 v3d_submit_put_jobs(struct v3d_submit *submit)
311 {
312 	for (int i = 0; i < submit->job_count; i++)
313 		v3d_job_put(submit->jobs[i]);
314 }
315 
316 static void
317 v3d_submit_cleanup_jobs(struct v3d_submit *submit)
318 {
319 	for (int i = 0; i < submit->job_count; i++)
320 		v3d_job_cleanup(submit->jobs[i]);
321 }
322 
323 static int
324 v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id)
325 {
326 	struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
327 	struct v3d_dev *v3d = submit->v3d;
328 	struct v3d_perfmon *perfmon;
329 
330 	if (!perfmon_id)
331 		return 0;
332 
333 	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
334 		if (v3d->global_perfmon)
335 			return -EAGAIN;
336 	}
337 
338 	perfmon = v3d_perfmon_find(v3d_priv, perfmon_id);
339 	if (!perfmon)
340 		return -ENOENT;
341 
342 	for (int i = 0; i < submit->job_count; i++) {
343 		submit->jobs[i]->perfmon = perfmon;
344 		if (i != 0)
345 			v3d_perfmon_get(perfmon);
346 	}
347 
348 	return 0;
349 }
350 
351 /*
352  * Prepare fences to enforce job serialization when a perfmon is active. A job
353  * that carries a non-global perfmon must wait for every job currently in-flight
354  * across all HW queues to finish, otherwise concurrent unrelated work on the
355  * same core would pollute the performance counters. Symmetrically, while such a
356  * job is still in-flight, all subsequently submitted jobs must wait for it.
357  *
358  * We don't serialize the jobs when using a global perfmon as it's expected to
359  * track concurrent activity from all jobs.
360  *
361  * Keeping track of the in-flight jobs costs a fence merge per job, so it is
362  * only done while at least one perfmon is alive, and is skipped for the jobs
363  * carrying no perfmon while a global perfmon is set. Jobs submitted while
364  * tracking is off go untracked and may overlap the first measured job.
365  */
366 static int
367 v3d_serialize_for_perfmon(struct v3d_job *job)
368 {
369 	struct v3d_dev *v3d = job->v3d;
370 	struct dma_fence *merged;
371 	int ret;
372 
373 	lockdep_assert_held(&v3d->sched_lock);
374 
375 	if (!atomic_read(&v3d->perfmon_state.nperfmons))
376 		return 0;
377 
378 	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
379 		if (!job->perfmon && v3d->global_perfmon)
380 			return 0;
381 	}
382 
383 	if (job->perfmon) {
384 		for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) {
385 			struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q];
386 
387 			if (!f || dma_fence_is_signaled(f))
388 				continue;
389 
390 			ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f));
391 			if (ret)
392 				return ret;
393 		}
394 	} else if (v3d->perfmon_state.fence &&
395 		   !dma_fence_is_signaled(v3d->perfmon_state.fence)) {
396 		ret = drm_sched_job_add_dependency(&job->base,
397 						   dma_fence_get(v3d->perfmon_state.fence));
398 		if (ret)
399 			return ret;
400 	}
401 
402 	/*
403 	 * Accumulate every in-flight job on this queue into one merged fence.
404 	 * A HW queue is fed by several scheduler entities (one per-fd), so jobs
405 	 * on it can complete out of order.
406 	 */
407 	merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue],
408 					job->done_fence);
409 	if (!merged)
410 		return -ENOMEM;
411 
412 	dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]);
413 	v3d->perfmon_state.last_hw_fence[job->queue] = merged;
414 
415 	if (job->perfmon) {
416 		dma_fence_put(v3d->perfmon_state.fence);
417 		v3d->perfmon_state.fence = dma_fence_get(job->done_fence);
418 	}
419 
420 	return 0;
421 }
422 
423 static void
424 v3d_submit_attach_object_fences(struct v3d_submit *submit)
425 {
426 	struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
427 
428 	/* The submission's last fence covers the entire submission. Attach it
429 	 * to every BO touched by any job in the submission.
430 	 */
431 	for (int i = 0; i < submit->job_count; i++) {
432 		struct v3d_job *job = submit->jobs[i];
433 
434 		for (int j = 0; j < job->bo_count; j++) {
435 			/* XXX: Use shared fences for read-only objects. */
436 			dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence,
437 					   DMA_RESV_USAGE_WRITE);
438 		}
439 	}
440 }
441 
442 static void
443 v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out,
444 			     struct v3d_submit_ext *se)
445 {
446 	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
447 	struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
448 
449 	/* Make sure single syncobj and multisync are mutually exclusive */
450 	WARN_ON_ONCE(sync_out && has_multisync);
451 
452 	/* Update the return sync object for the job */
453 	/* If it only supports a single signal semaphore*/
454 	if (!has_multisync) {
455 		if (sync_out) {
456 			drm_syncobj_replace_fence(sync_out, last_job->done_fence);
457 			drm_syncobj_put(sync_out);
458 		}
459 		return;
460 	}
461 
462 	/* If multiple semaphores extension is supported */
463 	if (se->out_sync_count) {
464 		for (int i = 0; i < se->out_sync_count; i++) {
465 			drm_syncobj_replace_fence(se->out_syncs[i].syncobj,
466 						  last_job->done_fence);
467 			drm_syncobj_put(se->out_syncs[i].syncobj);
468 		}
469 		kvfree(se->out_syncs);
470 	}
471 }
472 
473 static int
474 v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out,
475 		struct v3d_submit_ext *se)
476 {
477 	struct v3d_dev *v3d = submit->v3d;
478 	int ret = 0;
479 
480 	mutex_lock(&v3d->sched_lock);
481 
482 	for (int i = 0; i < submit->job_count; i++) {
483 		struct v3d_job *job = submit->jobs[i];
484 
485 		drm_sched_job_arm(&job->base);
486 		job->done_fence = dma_fence_get(&job->base.s_fence->finished);
487 
488 		/* put by scheduler job completion */
489 		kref_get(&job->refcount);
490 	}
491 
492 	for (int i = 1; i < submit->job_count; i++) {
493 		ret = drm_sched_job_add_dependency(&submit->jobs[i]->base,
494 						   dma_fence_get(submit->jobs[i - 1]->done_fence));
495 		if (ret)
496 			goto err;
497 	}
498 
499 	for (int i = 0; i < submit->job_count; i++) {
500 		ret = v3d_serialize_for_perfmon(submit->jobs[i]);
501 		if (ret)
502 			goto err;
503 	}
504 
505 	for (int i = 0; i < submit->job_count; i++)
506 		drm_sched_entity_push_job(&submit->jobs[i]->base);
507 
508 	mutex_unlock(&v3d->sched_lock);
509 
510 	v3d_submit_attach_object_fences(submit);
511 	v3d_submit_unlock_reservations(submit);
512 	v3d_submit_process_post_deps(submit, sync_out, se);
513 
514 	v3d_submit_put_jobs(submit);
515 
516 	return 0;
517 
518 err:
519 	/* Mark every armed job as failed so run_job() skips execution */
520 	for (int i = 0; i < submit->job_count; i++)
521 		dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret);
522 
523 	for (int i = 0; i < submit->job_count; i++)
524 		drm_sched_entity_push_job(&submit->jobs[i]->base);
525 
526 	mutex_unlock(&v3d->sched_lock);
527 
528 	v3d_submit_unlock_reservations(submit);
529 	v3d_submit_put_jobs(submit);
530 
531 	return ret;
532 }
533 
534 static int
535 v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit,
536 			   struct drm_v3d_submit_csd *args,
537 			   struct v3d_submit_ext *se)
538 {
539 	struct v3d_csd_job *job;
540 	struct v3d_job *clean_job;
541 	int ret;
542 
543 	job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD);
544 	if (IS_ERR(job))
545 		return PTR_ERR(job);
546 
547 	ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se);
548 	if (ret)
549 		return ret;
550 
551 	ret = v3d_lookup_bos(submit->file_priv, &job->base, args->bo_handles,
552 			     args->bo_handle_count);
553 	if (ret)
554 		return ret;
555 
556 	job->args = *args;
557 
558 	clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN);
559 	if (IS_ERR(clean_job))
560 		return PTR_ERR(clean_job);
561 
562 	return v3d_job_reference_bos(clean_job, &job->base);
563 }
564 
565 static void
566 v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se)
567 {
568 	unsigned int i;
569 
570 	if (sync_out)
571 		drm_syncobj_put(sync_out);
572 
573 	if (!(se && se->out_sync_count))
574 		return;
575 
576 	for (i = 0; i < se->out_sync_count; i++)
577 		drm_syncobj_put(se->out_syncs[i].syncobj);
578 	kvfree(se->out_syncs);
579 }
580 
581 static int
582 v3d_get_multisync_post_deps(struct drm_file *file_priv,
583 			    struct v3d_submit_ext *se,
584 			    u32 count, u64 handles)
585 {
586 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
587 	struct v3d_dev *v3d = v3d_priv->v3d;
588 	struct drm_v3d_sem __user *post_deps;
589 	int i, ret;
590 
591 	if (!count)
592 		return 0;
593 
594 	se->out_syncs = (struct v3d_submit_outsync *)
595 			kvmalloc_objs(struct v3d_submit_outsync, count);
596 	if (!se->out_syncs)
597 		return -ENOMEM;
598 
599 	post_deps = u64_to_user_ptr(handles);
600 
601 	for (i = 0; i < count; i++) {
602 		struct drm_v3d_sem out;
603 
604 		if (copy_from_user(&out, post_deps++, sizeof(out))) {
605 			ret = -EFAULT;
606 			drm_dbg(&v3d->drm, "Failed to copy post dep handles\n");
607 			goto fail;
608 		}
609 
610 		se->out_syncs[i].syncobj = drm_syncobj_find(file_priv,
611 							    out.handle);
612 		if (!se->out_syncs[i].syncobj) {
613 			ret = -EINVAL;
614 			goto fail;
615 		}
616 	}
617 	se->out_sync_count = count;
618 
619 	return 0;
620 
621 fail:
622 	for (i--; i >= 0; i--)
623 		drm_syncobj_put(se->out_syncs[i].syncobj);
624 	kvfree(se->out_syncs);
625 
626 	return ret;
627 }
628 
629 /* Get data for multiple binary semaphores synchronization. Parse syncobj
630  * to be signaled when job completes (out_sync).
631  */
632 static int
633 v3d_get_multisync_submit_deps(struct drm_file *file_priv,
634 			      struct drm_v3d_extension __user *ext,
635 			      struct v3d_submit_ext *se)
636 {
637 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
638 	struct v3d_dev *v3d = v3d_priv->v3d;
639 	struct drm_v3d_multi_sync multisync;
640 	int ret;
641 
642 	if (se->in_sync_count || se->out_sync_count) {
643 		drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job.");
644 		return -EINVAL;
645 	}
646 
647 	if (copy_from_user(&multisync, ext, sizeof(multisync)))
648 		return -EFAULT;
649 
650 	if (multisync.pad)
651 		return -EINVAL;
652 
653 	if (!multisync.in_sync_count && !multisync.out_sync_count) {
654 		drm_dbg(&v3d->drm, "Empty multisync extension\n");
655 		return -EINVAL;
656 	}
657 
658 	ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count,
659 					  multisync.out_syncs);
660 	if (ret)
661 		return ret;
662 
663 	se->in_sync_count = multisync.in_sync_count;
664 	se->in_syncs = multisync.in_syncs;
665 	se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC;
666 	se->wait_stage = multisync.wait_stage;
667 
668 	return 0;
669 }
670 
671 /* Returns false if the CPU job has an invalid configuration. */
672 static bool
673 v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job)
674 {
675 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
676 	struct v3d_dev *v3d = v3d_priv->v3d;
677 
678 	if (!job) {
679 		drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n");
680 		return false;
681 	}
682 
683 	if (job->job_type) {
684 		drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n");
685 		return false;
686 	}
687 
688 	return true;
689 }
690 
691 /* Get data for the indirect CSD job submission. */
692 static int
693 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv,
694 				struct drm_v3d_extension __user *ext,
695 				struct v3d_cpu_job *job)
696 {
697 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
698 	struct v3d_dev *v3d = v3d_priv->v3d;
699 	struct drm_v3d_indirect_csd indirect_csd;
700 	struct v3d_indirect_csd_info *info = &job->indirect_csd;
701 
702 	if (!v3d_validate_cpu_job(file_priv, job))
703 		return -EINVAL;
704 
705 	if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd)))
706 		return -EFAULT;
707 
708 	if (!v3d_has_csd(v3d)) {
709 		drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n");
710 		return -EINVAL;
711 	}
712 
713 	job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;
714 	info->args = indirect_csd.submit;
715 	info->offset = indirect_csd.offset;
716 	info->wg_size = indirect_csd.wg_size;
717 	memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets,
718 	       sizeof(indirect_csd.wg_uniform_offsets));
719 
720 	info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect);
721 	if (!info->indirect)
722 		return -ENOENT;
723 
724 	return 0;
725 }
726 
727 /* Get data for the query timestamp job submission. */
728 static int
729 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv,
730 				   struct drm_v3d_extension __user *ext,
731 				   struct v3d_cpu_job *job)
732 {
733 	u32 __user *offsets, *syncs;
734 	struct drm_v3d_timestamp_query timestamp;
735 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
736 	unsigned int i;
737 	int err;
738 
739 	if (!v3d_validate_cpu_job(file_priv, job))
740 		return -EINVAL;
741 
742 	if (copy_from_user(&timestamp, ext, sizeof(timestamp)))
743 		return -EFAULT;
744 
745 	if (timestamp.pad)
746 		return -EINVAL;
747 
748 	job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY;
749 
750 	query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
751 					    timestamp.count);
752 	if (!query_info->queries)
753 		return -ENOMEM;
754 
755 	offsets = u64_to_user_ptr(timestamp.offsets);
756 	syncs = u64_to_user_ptr(timestamp.syncs);
757 
758 	for (i = 0; i < timestamp.count; i++) {
759 		u32 offset, sync;
760 
761 		if (get_user(offset, offsets++)) {
762 			err = -EFAULT;
763 			goto error;
764 		}
765 
766 		query_info->queries[i].offset = offset;
767 
768 		if (get_user(sync, syncs++)) {
769 			err = -EFAULT;
770 			goto error;
771 		}
772 
773 		query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
774 								  sync);
775 		if (!query_info->queries[i].syncobj) {
776 			err = -ENOENT;
777 			goto error;
778 		}
779 	}
780 	query_info->count = timestamp.count;
781 
782 	return 0;
783 
784 error:
785 	v3d_timestamp_query_info_free(&job->timestamp_query, i);
786 	return err;
787 }
788 
789 static int
790 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv,
791 				   struct drm_v3d_extension __user *ext,
792 				   struct v3d_cpu_job *job)
793 {
794 	u32 __user *syncs;
795 	struct drm_v3d_reset_timestamp_query reset;
796 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
797 	unsigned int i;
798 	int err;
799 
800 	if (!v3d_validate_cpu_job(file_priv, job))
801 		return -EINVAL;
802 
803 	if (copy_from_user(&reset, ext, sizeof(reset)))
804 		return -EFAULT;
805 
806 	job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY;
807 
808 	query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
809 					    reset.count);
810 	if (!query_info->queries)
811 		return -ENOMEM;
812 
813 	syncs = u64_to_user_ptr(reset.syncs);
814 
815 	for (i = 0; i < reset.count; i++) {
816 		u32 sync;
817 
818 		query_info->queries[i].offset = reset.offset + 8 * i;
819 
820 		if (get_user(sync, syncs++)) {
821 			err = -EFAULT;
822 			goto error;
823 		}
824 
825 		query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
826 								  sync);
827 		if (!query_info->queries[i].syncobj) {
828 			err = -ENOENT;
829 			goto error;
830 		}
831 	}
832 	query_info->count = reset.count;
833 
834 	return 0;
835 
836 error:
837 	v3d_timestamp_query_info_free(&job->timestamp_query, i);
838 	return err;
839 }
840 
841 /* Get data for the copy timestamp query results job submission. */
842 static int
843 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv,
844 				      struct drm_v3d_extension __user *ext,
845 				      struct v3d_cpu_job *job)
846 {
847 	u32 __user *offsets, *syncs;
848 	struct drm_v3d_copy_timestamp_query copy;
849 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
850 	unsigned int i;
851 	int err;
852 
853 	if (!v3d_validate_cpu_job(file_priv, job))
854 		return -EINVAL;
855 
856 	if (copy_from_user(&copy, ext, sizeof(copy)))
857 		return -EFAULT;
858 
859 	if (copy.pad)
860 		return -EINVAL;
861 
862 	job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY;
863 
864 	query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
865 					    copy.count);
866 	if (!query_info->queries)
867 		return -ENOMEM;
868 
869 	offsets = u64_to_user_ptr(copy.offsets);
870 	syncs = u64_to_user_ptr(copy.syncs);
871 
872 	for (i = 0; i < copy.count; i++) {
873 		u32 offset, sync;
874 
875 		if (get_user(offset, offsets++)) {
876 			err = -EFAULT;
877 			goto error;
878 		}
879 
880 		query_info->queries[i].offset = offset;
881 
882 		if (get_user(sync, syncs++)) {
883 			err = -EFAULT;
884 			goto error;
885 		}
886 
887 		query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
888 								  sync);
889 		if (!query_info->queries[i].syncobj) {
890 			err = -ENOENT;
891 			goto error;
892 		}
893 	}
894 	query_info->count = copy.count;
895 
896 	job->copy.do_64bit = copy.do_64bit;
897 	job->copy.do_partial = copy.do_partial;
898 	job->copy.availability_bit = copy.availability_bit;
899 	job->copy.offset = copy.offset;
900 	job->copy.stride = copy.stride;
901 
902 	return 0;
903 
904 error:
905 	v3d_timestamp_query_info_free(&job->timestamp_query, i);
906 	return err;
907 }
908 
909 static int
910 v3d_copy_query_info(struct v3d_performance_query_info *query_info,
911 		    unsigned int count,
912 		    unsigned int nperfmons,
913 		    u32 __user *syncs,
914 		    u64 __user *kperfmon_ids,
915 		    struct drm_file *file_priv)
916 {
917 	unsigned int i, j;
918 	int err;
919 
920 	for (i = 0; i < count; i++) {
921 		struct v3d_performance_query *query = &query_info->queries[i];
922 		u32 __user *ids_pointer;
923 		u32 sync, id;
924 		u64 ids;
925 
926 		if (get_user(sync, syncs++)) {
927 			err = -EFAULT;
928 			goto error;
929 		}
930 
931 		if (get_user(ids, kperfmon_ids++)) {
932 			err = -EFAULT;
933 			goto error;
934 		}
935 
936 		query->kperfmon_ids =
937 			kvmalloc_array(nperfmons,
938 				       sizeof(struct v3d_performance_query *),
939 				       GFP_KERNEL);
940 		if (!query->kperfmon_ids) {
941 			err = -ENOMEM;
942 			goto error;
943 		}
944 
945 		ids_pointer = u64_to_user_ptr(ids);
946 
947 		for (j = 0; j < nperfmons; j++) {
948 			if (get_user(id, ids_pointer++)) {
949 				kvfree(query->kperfmon_ids);
950 				err = -EFAULT;
951 				goto error;
952 			}
953 
954 			query->kperfmon_ids[j] = id;
955 		}
956 
957 		query->syncobj = drm_syncobj_find(file_priv, sync);
958 		if (!query->syncobj) {
959 			kvfree(query->kperfmon_ids);
960 			err = -ENOENT;
961 			goto error;
962 		}
963 	}
964 
965 	return 0;
966 
967 error:
968 	v3d_performance_query_info_free(query_info, i);
969 	return err;
970 }
971 
972 static int
973 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv,
974 				     struct drm_v3d_extension __user *ext,
975 				     struct v3d_cpu_job *job)
976 {
977 	struct v3d_performance_query_info *query_info = &job->performance_query;
978 	struct drm_v3d_reset_performance_query reset;
979 	int err;
980 
981 	if (!v3d_validate_cpu_job(file_priv, job))
982 		return -EINVAL;
983 
984 	if (copy_from_user(&reset, ext, sizeof(reset)))
985 		return -EFAULT;
986 
987 	job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY;
988 
989 	query_info->queries =
990 		kvmalloc_objs(struct v3d_performance_query, reset.count);
991 	if (!query_info->queries)
992 		return -ENOMEM;
993 
994 	err = v3d_copy_query_info(query_info,
995 				  reset.count,
996 				  reset.nperfmons,
997 				  u64_to_user_ptr(reset.syncs),
998 				  u64_to_user_ptr(reset.kperfmon_ids),
999 				  file_priv);
1000 	if (err)
1001 		return err;
1002 
1003 	query_info->count = reset.count;
1004 	query_info->nperfmons = reset.nperfmons;
1005 
1006 	return 0;
1007 }
1008 
1009 static int
1010 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv,
1011 					  struct drm_v3d_extension __user *ext,
1012 					  struct v3d_cpu_job *job)
1013 {
1014 	struct v3d_performance_query_info *query_info = &job->performance_query;
1015 	struct drm_v3d_copy_performance_query copy;
1016 	int err;
1017 
1018 	if (!v3d_validate_cpu_job(file_priv, job))
1019 		return -EINVAL;
1020 
1021 	if (copy_from_user(&copy, ext, sizeof(copy)))
1022 		return -EFAULT;
1023 
1024 	if (copy.pad)
1025 		return -EINVAL;
1026 
1027 	job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY;
1028 
1029 	query_info->queries =
1030 		kvmalloc_objs(struct v3d_performance_query, copy.count);
1031 	if (!query_info->queries)
1032 		return -ENOMEM;
1033 
1034 	err = v3d_copy_query_info(query_info,
1035 				  copy.count,
1036 				  copy.nperfmons,
1037 				  u64_to_user_ptr(copy.syncs),
1038 				  u64_to_user_ptr(copy.kperfmon_ids),
1039 				  file_priv);
1040 	if (err)
1041 		return err;
1042 
1043 	query_info->count = copy.count;
1044 	query_info->nperfmons = copy.nperfmons;
1045 	query_info->ncounters = copy.ncounters;
1046 
1047 	job->copy.do_64bit = copy.do_64bit;
1048 	job->copy.do_partial = copy.do_partial;
1049 	job->copy.availability_bit = copy.availability_bit;
1050 	job->copy.offset = copy.offset;
1051 	job->copy.stride = copy.stride;
1052 
1053 	return 0;
1054 }
1055 
1056 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data
1057  * according to the extension id (name).
1058  */
1059 static int
1060 v3d_get_extensions(struct drm_file *file_priv,
1061 		   u64 ext_handles,
1062 		   struct v3d_submit_ext *se,
1063 		   struct v3d_cpu_job *job)
1064 {
1065 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
1066 	struct v3d_dev *v3d = v3d_priv->v3d;
1067 	struct drm_v3d_extension __user *user_ext;
1068 	int ret;
1069 
1070 	user_ext = u64_to_user_ptr(ext_handles);
1071 	while (user_ext) {
1072 		struct drm_v3d_extension ext;
1073 
1074 		if (copy_from_user(&ext, user_ext, sizeof(ext))) {
1075 			drm_dbg(&v3d->drm, "Failed to copy submit extension\n");
1076 			return -EFAULT;
1077 		}
1078 
1079 		switch (ext.id) {
1080 		case DRM_V3D_EXT_ID_MULTI_SYNC:
1081 			ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se);
1082 			break;
1083 		case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD:
1084 			ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job);
1085 			break;
1086 		case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY:
1087 			ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job);
1088 			break;
1089 		case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY:
1090 			ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job);
1091 			break;
1092 		case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY:
1093 			ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job);
1094 			break;
1095 		case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY:
1096 			ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job);
1097 			break;
1098 		case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY:
1099 			ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job);
1100 			break;
1101 		default:
1102 			drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id);
1103 			return -EINVAL;
1104 		}
1105 
1106 		if (ret)
1107 			return ret;
1108 
1109 		user_ext = u64_to_user_ptr(ext.next);
1110 	}
1111 
1112 	return 0;
1113 }
1114 
1115 /**
1116  * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
1117  * @dev: DRM device
1118  * @data: ioctl argument
1119  * @file_priv: DRM file for this fd
1120  *
1121  * This is the main entrypoint for userspace to submit a 3D frame to
1122  * the GPU.  Userspace provides the binner command list (if
1123  * applicable), and the kernel sets up the render command list to draw
1124  * to the framebuffer described in the ioctl, using the command lists
1125  * that the 3D engine's binner will produce.
1126  */
1127 int
1128 v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
1129 		    struct drm_file *file_priv)
1130 {
1131 	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1132 	struct drm_v3d_submit_cl *args = data;
1133 	struct drm_syncobj *sync_out = NULL;
1134 	struct v3d_submit_ext se = {0};
1135 	struct v3d_bin_job *bin = NULL;
1136 	struct v3d_render_job *render;
1137 	struct v3d_job *clean_job;
1138 	int ret;
1139 
1140 	trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end);
1141 
1142 	if (args->pad)
1143 		return -EINVAL;
1144 
1145 	if (args->flags &&
1146 	    args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE |
1147 			    DRM_V3D_SUBMIT_EXTENSION)) {
1148 		drm_dbg(dev, "invalid flags: %d\n", args->flags);
1149 		return -EINVAL;
1150 	}
1151 
1152 	if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1153 		ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1154 		if (ret) {
1155 			drm_dbg(dev, "Failed to get extensions.\n");
1156 			return ret;
1157 		}
1158 	}
1159 
1160 	/* If multisync is configured, give priority to it and ignore out_sync. */
1161 	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1162 		sync_out = drm_syncobj_find(file_priv, args->out_sync);
1163 		if (!sync_out)
1164 			return -ENOENT;
1165 	}
1166 
1167 	if (args->bcl_start != args->bcl_end) {
1168 		bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN);
1169 		if (IS_ERR(bin)) {
1170 			ret = PTR_ERR(bin);
1171 			goto fail;
1172 		}
1173 
1174 		bin->start = args->bcl_start;
1175 		bin->end = args->bcl_end;
1176 		bin->qma = args->qma;
1177 		bin->qms = args->qms;
1178 		bin->qts = args->qts;
1179 
1180 		ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl,
1181 					   &se);
1182 		if (ret)
1183 			goto fail;
1184 	}
1185 
1186 	render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER);
1187 	if (IS_ERR(render)) {
1188 		ret = PTR_ERR(render);
1189 		goto fail;
1190 	}
1191 
1192 	INIT_LIST_HEAD(&render->unref_list);
1193 	render->start = args->rcl_start;
1194 	render->end = args->rcl_end;
1195 
1196 	if (bin)
1197 		bin->render = render;
1198 
1199 	ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se);
1200 	if (ret)
1201 		goto fail;
1202 
1203 	/*
1204 	 * We don't associate the BOs with the BIN job. Fences are only
1205 	 * attached to the last job in the submission chain, and BIN jobs
1206 	 * don't need implicit dependencies since depending on results from
1207 	 * another context is not a realistic scenario for binning.
1208 	 */
1209 	ret = v3d_lookup_bos(submit.file_priv, &render->base,
1210 			     args->bo_handles, args->bo_handle_count);
1211 	if (ret)
1212 		goto fail;
1213 
1214 	if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
1215 		clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN);
1216 		if (IS_ERR(clean_job)) {
1217 			ret = PTR_ERR(clean_job);
1218 			goto fail;
1219 		}
1220 
1221 		ret = v3d_job_reference_bos(clean_job, &render->base);
1222 		if (ret)
1223 			goto fail;
1224 	}
1225 
1226 	ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
1227 	if (ret)
1228 		goto fail;
1229 
1230 	ret = v3d_submit_lock_reservations(&submit);
1231 	if (ret)
1232 		goto fail;
1233 
1234 	ret = v3d_submit_jobs(&submit, sync_out, &se);
1235 	if (ret)
1236 		goto fail_submit;
1237 
1238 	return 0;
1239 
1240 fail:
1241 	v3d_submit_cleanup_jobs(&submit);
1242 fail_submit:
1243 	v3d_submit_put_post_deps(sync_out, &se);
1244 
1245 	return ret;
1246 }
1247 
1248 /**
1249  * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
1250  * @dev: DRM device
1251  * @data: ioctl argument
1252  * @file_priv: DRM file for this fd
1253  *
1254  * Userspace provides the register setup for the TFU, which we don't
1255  * need to validate since the TFU is behind the MMU.
1256  */
1257 int
1258 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
1259 		     struct drm_file *file_priv)
1260 {
1261 	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1262 	struct drm_v3d_submit_tfu *args = data;
1263 	struct drm_syncobj *sync_out = NULL;
1264 	struct v3d_submit_ext se = {0};
1265 	struct v3d_tfu_job *job;
1266 	int ret = 0;
1267 
1268 	trace_v3d_submit_tfu_ioctl(dev, args->iia);
1269 
1270 	if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1271 		drm_dbg(dev, "invalid flags: %d\n", args->flags);
1272 		return -EINVAL;
1273 	}
1274 
1275 	if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1276 		ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1277 		if (ret) {
1278 			drm_dbg(dev, "Failed to get extensions.\n");
1279 			return ret;
1280 		}
1281 	}
1282 
1283 	/* If multisync is configured, give priority to it and ignore out_sync. */
1284 	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1285 		sync_out = drm_syncobj_find(file_priv, args->out_sync);
1286 		if (!sync_out)
1287 			return -ENOENT;
1288 	}
1289 
1290 	job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU);
1291 	if (IS_ERR(job)) {
1292 		ret = PTR_ERR(job);
1293 		goto fail;
1294 	}
1295 
1296 	ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se);
1297 	if (ret)
1298 		goto fail;
1299 
1300 	job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles));
1301 	if (!job->base.bo) {
1302 		ret = -ENOMEM;
1303 		goto fail;
1304 	}
1305 
1306 	job->args = *args;
1307 
1308 	for (job->base.bo_count = 0;
1309 	     job->base.bo_count < ARRAY_SIZE(args->bo_handles);
1310 	     job->base.bo_count++) {
1311 		struct drm_gem_object *bo;
1312 
1313 		if (!args->bo_handles[job->base.bo_count])
1314 			break;
1315 
1316 		bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]);
1317 		if (!bo) {
1318 			drm_dbg(dev, "Failed to look up GEM BO %d: %d\n",
1319 				job->base.bo_count,
1320 				args->bo_handles[job->base.bo_count]);
1321 			ret = -ENOENT;
1322 			goto fail;
1323 		}
1324 		job->base.bo[job->base.bo_count] = bo;
1325 	}
1326 
1327 	ret = v3d_submit_lock_reservations(&submit);
1328 	if (ret)
1329 		goto fail;
1330 
1331 	ret = v3d_submit_jobs(&submit, sync_out, &se);
1332 	if (ret)
1333 		goto fail_submit;
1334 
1335 	return 0;
1336 
1337 fail:
1338 	v3d_submit_cleanup_jobs(&submit);
1339 fail_submit:
1340 	v3d_submit_put_post_deps(sync_out, &se);
1341 
1342 	return ret;
1343 }
1344 
1345 /**
1346  * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D.
1347  * @dev: DRM device
1348  * @data: ioctl argument
1349  * @file_priv: DRM file for this fd
1350  *
1351  * Userspace provides the register setup for the CSD, which we don't
1352  * need to validate since the CSD is behind the MMU.
1353  */
1354 int
1355 v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
1356 		     struct drm_file *file_priv)
1357 {
1358 	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1359 	struct drm_v3d_submit_csd *args = data;
1360 	struct drm_syncobj *sync_out = NULL;
1361 	struct v3d_submit_ext se = {0};
1362 	int ret;
1363 
1364 	trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]);
1365 
1366 	if (args->pad)
1367 		return -EINVAL;
1368 
1369 	if (!v3d_has_csd(submit.v3d)) {
1370 		drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n");
1371 		return -EINVAL;
1372 	}
1373 
1374 	if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1375 		drm_dbg(dev, "invalid flags: %d\n", args->flags);
1376 		return -EINVAL;
1377 	}
1378 
1379 	if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1380 		ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1381 		if (ret) {
1382 			drm_dbg(dev, "Failed to get extensions.\n");
1383 			return ret;
1384 		}
1385 	}
1386 
1387 	/* If multisync is configured, give priority to it and ignore out_sync. */
1388 	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1389 		sync_out = drm_syncobj_find(file_priv, args->out_sync);
1390 		if (!sync_out)
1391 			return -ENOENT;
1392 	}
1393 
1394 	ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se);
1395 	if (ret)
1396 		goto fail;
1397 
1398 	ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
1399 	if (ret)
1400 		goto fail;
1401 
1402 	ret = v3d_submit_lock_reservations(&submit);
1403 	if (ret)
1404 		goto fail;
1405 
1406 	ret = v3d_submit_jobs(&submit, sync_out, &se);
1407 	if (ret)
1408 		goto fail_submit;
1409 
1410 	return 0;
1411 
1412 fail:
1413 	v3d_submit_cleanup_jobs(&submit);
1414 fail_submit:
1415 	v3d_submit_put_post_deps(sync_out, &se);
1416 
1417 	return ret;
1418 }
1419 
1420 static const unsigned int cpu_job_bo_handle_count[] = {
1421 	[V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1,
1422 	[V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1,
1423 	[V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1,
1424 	[V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2,
1425 	[V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0,
1426 	[V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1,
1427 };
1428 
1429 /* Reject offset + (count - 1) * stride + write_size if it leaves the BO. */
1430 static int
1431 v3d_check_copy_extent(struct drm_device *dev, size_t bo_size,
1432 		      u32 offset, u32 stride, u32 count, u64 write_size)
1433 {
1434 	u64 last;
1435 
1436 	if (!count)
1437 		return 0;
1438 
1439 	/*
1440 	 * The executors walk a u8 * cursor, so the furthest written byte is
1441 	 * offset + (count - 1) * stride + write_size, matching the pointer
1442 	 * arithmetic in v3d_copy_query_results()/v3d_copy_performance_query().
1443 	 * (count - 1) * stride is a u32 * u32 product that is exact in u64,
1444 	 * and offset + write_size stays far below the u64 range, so a single
1445 	 * overflow check guards the total.
1446 	 */
1447 	last = write_size + offset;
1448 	if (check_add_overflow((u64)(count - 1) * stride, last, &last) ||
1449 	    last > bo_size) {
1450 		drm_dbg(dev, "CPU job copy buffer exceeds the destination BO.\n");
1451 		return -EINVAL;
1452 	}
1453 
1454 	return 0;
1455 }
1456 
1457 /* Reject a query CPU job whose writes would land outside their BO. */
1458 static int
1459 v3d_cpu_job_bounds_check(struct v3d_cpu_job *job)
1460 {
1461 	struct drm_device *dev = &job->base.v3d->drm;
1462 	struct v3d_timestamp_query_info *tquery = &job->timestamp_query;
1463 	struct v3d_copy_query_results_info *copy = &job->copy;
1464 	u32 elem = copy->do_64bit ? sizeof(u64) : sizeof(u32);
1465 	struct v3d_bo *dst, *src;
1466 	u64 slots, write_size;
1467 	u32 i;
1468 
1469 	switch (job->job_type) {
1470 	case V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY:
1471 	case V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY:
1472 		/* Each query writes one u64 timestamp slot into bo[0]. */
1473 		dst = to_v3d_bo(job->base.bo[0]);
1474 
1475 		for (i = 0; i < tquery->count; i++) {
1476 			if ((u64)tquery->queries[i].offset + sizeof(u64) >
1477 			    dst->base.base.size)
1478 				goto err_range;
1479 		}
1480 		return 0;
1481 	case V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY:
1482 		/* Copies one u64 per query from bo[1] into bo[0]. */
1483 		dst = to_v3d_bo(job->base.bo[0]);
1484 		src = to_v3d_bo(job->base.bo[1]);
1485 
1486 		for (i = 0; i < tquery->count; i++) {
1487 			if ((u64)tquery->queries[i].offset + sizeof(u64) >
1488 			    src->base.base.size)
1489 				goto err_range;
1490 		}
1491 
1492 		write_size = (copy->availability_bit ? 2 : 1) * elem;
1493 		return v3d_check_copy_extent(dev, dst->base.base.size,
1494 					     copy->offset, copy->stride,
1495 					     tquery->count, write_size);
1496 	case V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY:
1497 		/*
1498 		 * Each query writes nperfmons * DRM_V3D_MAX_PERF_COUNTERS
1499 		 * counter slots into bo[0], plus an availability slot at
1500 		 * index ncounters. nperfmons and ncounters are user values,
1501 		 * so the slot count is computed overflow-safe.
1502 		 */
1503 		dst = to_v3d_bo(job->base.bo[0]);
1504 
1505 		slots = (u64)job->performance_query.nperfmons *
1506 			DRM_V3D_MAX_PERF_COUNTERS;
1507 		if (copy->availability_bit)
1508 			slots = max(slots,
1509 				    (u64)job->performance_query.ncounters + 1);
1510 
1511 		write_size = slots * elem;
1512 		return v3d_check_copy_extent(dev, dst->base.base.size,
1513 					     copy->offset, copy->stride,
1514 					     job->performance_query.count,
1515 					     write_size);
1516 	case V3D_CPU_JOB_TYPE_INDIRECT_CSD: {
1517 		struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;
1518 
1519 		/* 3 is the three dimensions (x, y, z) of the workgroup counts. */
1520 		src = to_v3d_bo(job->base.bo[0]);
1521 		if ((u64)indirect_csd->offset + 3 * sizeof(u32) >
1522 		    src->base.base.size)
1523 			goto err_range;
1524 
1525 		dst = to_v3d_bo(indirect_csd->indirect);
1526 		for (i = 0; i < 3; i++) {
1527 			u32 uidx = indirect_csd->wg_uniform_offsets[i];
1528 
1529 			/*
1530 			 * 0xffffffff means "skip this rewrite", so the exec
1531 			 * path never writes that index and it needs no check.
1532 			 */
1533 			if (uidx != 0xffffffff &&
1534 			    (u64)uidx * sizeof(u32) + sizeof(u32) >
1535 			    dst->base.base.size)
1536 				goto err_range;
1537 		}
1538 		return 0;
1539 	}
1540 	default:
1541 		return 0;
1542 	}
1543 
1544 err_range:
1545 	drm_dbg(dev, "CPU job query offset exceeds the BO.\n");
1546 	return -EINVAL;
1547 }
1548 
1549 /**
1550  * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
1551  * @dev: DRM device
1552  * @data: ioctl argument
1553  * @file_priv: DRM file for this fd
1554  *
1555  * Userspace specifies the CPU job type and data required to perform its
1556  * operations through the drm_v3d_extension struct.
1557  */
1558 int
1559 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
1560 		     struct drm_file *file_priv)
1561 {
1562 	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1563 	struct drm_v3d_submit_cpu *args = data;
1564 	struct v3d_submit_ext se = {0};
1565 	struct v3d_cpu_job *cpu_job = NULL;
1566 	int ret;
1567 
1568 	if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1569 		drm_dbg(dev, "Invalid flags: %d\n", args->flags);
1570 		return -EINVAL;
1571 	}
1572 
1573 	cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU);
1574 	if (IS_ERR(cpu_job))
1575 		return PTR_ERR(cpu_job);
1576 
1577 	if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1578 		ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job);
1579 		if (ret) {
1580 			drm_dbg(dev, "Failed to get extensions.\n");
1581 			goto fail;
1582 		}
1583 	}
1584 
1585 	/* Every CPU job must have a CPU job user extension */
1586 	if (!cpu_job->job_type) {
1587 		drm_dbg(dev, "CPU job must have a CPU job user extension.\n");
1588 		ret = -EINVAL;
1589 		goto fail;
1590 	}
1591 
1592 	if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) {
1593 		drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n");
1594 		ret = -EINVAL;
1595 		goto fail;
1596 	}
1597 
1598 	trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type);
1599 
1600 	ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se);
1601 	if (ret)
1602 		goto fail;
1603 
1604 	/* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends
1605 	 * the CSD and clean jobs in the case of indirect CSD job.
1606 	 */
1607 	if (args->bo_handle_count) {
1608 		ret = v3d_lookup_bos(submit.file_priv, &cpu_job->base,
1609 				     args->bo_handles, args->bo_handle_count);
1610 		if (ret)
1611 			goto fail;
1612 
1613 		ret = v3d_cpu_job_bounds_check(cpu_job);
1614 		if (ret)
1615 			goto fail;
1616 	}
1617 
1618 	if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) {
1619 		ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args,
1620 						 NULL);
1621 		if (ret)
1622 			goto fail;
1623 
1624 		/* The CSD job was appended at jobs[1] */
1625 		if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) {
1626 			ret = -EINVAL;
1627 			goto fail;
1628 		}
1629 
1630 		cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job,
1631 							 base);
1632 	}
1633 
1634 	ret = v3d_submit_lock_reservations(&submit);
1635 	if (ret)
1636 		goto fail;
1637 
1638 	ret = v3d_submit_jobs(&submit, NULL, &se);
1639 	if (ret)
1640 		goto fail_submit;
1641 
1642 	return 0;
1643 
1644 fail:
1645 	v3d_submit_cleanup_jobs(&submit);
1646 fail_submit:
1647 	v3d_submit_put_post_deps(NULL, &se);
1648 
1649 	return ret;
1650 }
1651