xref: /linux/drivers/gpu/drm/panthor/panthor_drv.c (revision 128c8f96eb8638c060cd3532dc394d046ce64fe1)
1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4 /* Copyright 2019 Collabora ltd. */
5 
6 #ifdef CONFIG_ARM_ARCH_TIMER
7 #include <asm/arch_timer.h>
8 #endif
9 
10 #include <linux/list.h>
11 #include <linux/module.h>
12 #include <linux/of_platform.h>
13 #include <linux/pagemap.h>
14 #include <linux/platform_device.h>
15 #include <linux/pm_runtime.h>
16 #include <linux/time64.h>
17 
18 #include <drm/drm_auth.h>
19 #include <drm/drm_debugfs.h>
20 #include <drm/drm_drv.h>
21 #include <drm/drm_exec.h>
22 #include <drm/drm_ioctl.h>
23 #include <drm/drm_syncobj.h>
24 #include <drm/drm_utils.h>
25 #include <drm/gpu_scheduler.h>
26 #include <drm/panthor_drm.h>
27 
28 #include "panthor_device.h"
29 #include "panthor_fw.h"
30 #include "panthor_gem.h"
31 #include "panthor_gpu.h"
32 #include "panthor_heap.h"
33 #include "panthor_mmu.h"
34 #include "panthor_regs.h"
35 #include "panthor_sched.h"
36 
37 /**
38  * DOC: user <-> kernel object copy helpers.
39  */
40 
41 /**
42  * panthor_set_uobj() - Copy kernel object to user object.
43  * @usr_ptr: Users pointer.
44  * @usr_size: Size of the user object.
45  * @min_size: Minimum size for this object.
46  * @kern_size: Size of the kernel object.
47  * @in: Address of the kernel object to copy.
48  *
49  * Helper automating kernel -> user object copies.
50  *
51  * Don't use this function directly, use PANTHOR_UOBJ_SET() instead.
52  *
53  * Return: 0 on success, a negative error code otherwise.
54  */
55 static int
panthor_set_uobj(u64 usr_ptr,u32 usr_size,u32 min_size,u32 kern_size,const void * in)56 panthor_set_uobj(u64 usr_ptr, u32 usr_size, u32 min_size, u32 kern_size, const void *in)
57 {
58 	/* User size shouldn't be smaller than the minimal object size. */
59 	if (usr_size < min_size)
60 		return -EINVAL;
61 
62 	if (copy_to_user(u64_to_user_ptr(usr_ptr), in, min_t(u32, usr_size, kern_size)))
63 		return -EFAULT;
64 
65 	/* When the kernel object is smaller than the user object, we fill the gap with
66 	 * zeros.
67 	 */
68 	if (usr_size > kern_size &&
69 	    clear_user(u64_to_user_ptr(usr_ptr + kern_size), usr_size - kern_size)) {
70 		return -EFAULT;
71 	}
72 
73 	return 0;
74 }
75 
76 /**
77  * panthor_get_uobj_array() - Copy a user object array into a kernel accessible object array.
78  * @in: The object array to copy.
79  * @min_stride: Minimum array stride.
80  * @obj_size: Kernel object size.
81  *
82  * Helper automating user -> kernel object copies.
83  *
84  * Don't use this function directly, use PANTHOR_UOBJ_GET_ARRAY() instead.
85  *
86  * Return: newly allocated object array or an ERR_PTR on error.
87  */
88 static void *
panthor_get_uobj_array(const struct drm_panthor_obj_array * in,u32 min_stride,u32 obj_size)89 panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride,
90 		       u32 obj_size)
91 {
92 	int ret = 0;
93 	void *out_alloc;
94 
95 	if (!in->count)
96 		return NULL;
97 
98 	/* User stride must be at least the minimum object size, otherwise it might
99 	 * lack useful information.
100 	 */
101 	if (in->stride < min_stride)
102 		return ERR_PTR(-EINVAL);
103 
104 	out_alloc = kvmalloc_array(in->count, obj_size, GFP_KERNEL);
105 	if (!out_alloc)
106 		return ERR_PTR(-ENOMEM);
107 
108 	if (obj_size == in->stride) {
109 		/* Fast path when user/kernel have the same uAPI header version. */
110 		if (copy_from_user(out_alloc, u64_to_user_ptr(in->array),
111 				   (unsigned long)obj_size * in->count))
112 			ret = -EFAULT;
113 	} else {
114 		void __user *in_ptr = u64_to_user_ptr(in->array);
115 		void *out_ptr = out_alloc;
116 
117 		/* If the sizes differ, we need to copy elements one by one. */
118 		for (u32 i = 0; i < in->count; i++) {
119 			ret = copy_struct_from_user(out_ptr, obj_size, in_ptr, in->stride);
120 			if (ret)
121 				break;
122 
123 			out_ptr += obj_size;
124 			in_ptr += in->stride;
125 		}
126 	}
127 
128 	if (ret) {
129 		kvfree(out_alloc);
130 		return ERR_PTR(ret);
131 	}
132 
133 	return out_alloc;
134 }
135 
136 /**
137  * PANTHOR_UOBJ_MIN_SIZE_INTERNAL() - Get the minimum user object size
138  * @_typename: Object type.
139  * @_last_mandatory_field: Last mandatory field.
140  *
141  * Get the minimum user object size based on the last mandatory field name,
142  * A.K.A, the name of the last field of the structure at the time this
143  * structure was added to the uAPI.
144  *
145  * Don't use directly, use PANTHOR_UOBJ_DECL() instead.
146  */
147 #define PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field) \
148 	(offsetof(_typename, _last_mandatory_field) + \
149 	 sizeof(((_typename *)NULL)->_last_mandatory_field))
150 
151 /**
152  * PANTHOR_UOBJ_DECL() - Declare a new uAPI object whose subject to
153  * evolutions.
154  * @_typename: Object type.
155  * @_last_mandatory_field: Last mandatory field.
156  *
157  * Should be used to extend the PANTHOR_UOBJ_MIN_SIZE() list.
158  */
159 #define PANTHOR_UOBJ_DECL(_typename, _last_mandatory_field) \
160 	_typename : PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field)
161 
162 /**
163  * PANTHOR_UOBJ_MIN_SIZE() - Get the minimum size of a given uAPI object
164  * @_obj_name: Object to get the minimum size of.
165  *
166  * Don't use this macro directly, it's automatically called by
167  * PANTHOR_UOBJ_{SET,GET_ARRAY}().
168  */
169 #define PANTHOR_UOBJ_MIN_SIZE(_obj_name) \
170 	_Generic(_obj_name, \
171 		 PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \
172 		 PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \
173 		 PANTHOR_UOBJ_DECL(struct drm_panthor_timestamp_info, current_timestamp), \
174 		 PANTHOR_UOBJ_DECL(struct drm_panthor_group_priorities_info, pad), \
175 		 PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \
176 		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \
177 		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \
178 		 PANTHOR_UOBJ_DECL(struct drm_panthor_vm_bind_op, syncs))
179 
180 /**
181  * PANTHOR_UOBJ_SET() - Copy a kernel object to a user object.
182  * @_dest_usr_ptr: User pointer to copy to.
183  * @_usr_size: Size of the user object.
184  * @_src_obj: Kernel object to copy (not a pointer).
185  *
186  * Return: 0 on success, a negative error code otherwise.
187  */
188 #define PANTHOR_UOBJ_SET(_dest_usr_ptr, _usr_size, _src_obj) \
189 	panthor_set_uobj(_dest_usr_ptr, _usr_size, \
190 			 PANTHOR_UOBJ_MIN_SIZE(_src_obj), \
191 			 sizeof(_src_obj), &(_src_obj))
192 
193 /**
194  * PANTHOR_UOBJ_GET_ARRAY() - Copy a user object array to a kernel accessible
195  * object array.
196  * @_dest_array: Local variable that will hold the newly allocated kernel
197  * object array.
198  * @_uobj_array: The drm_panthor_obj_array object describing the user object
199  * array.
200  *
201  * Return: 0 on success, a negative error code otherwise.
202  */
203 #define PANTHOR_UOBJ_GET_ARRAY(_dest_array, _uobj_array) \
204 	({ \
205 		typeof(_dest_array) _tmp; \
206 		_tmp = panthor_get_uobj_array(_uobj_array, \
207 					      PANTHOR_UOBJ_MIN_SIZE((_dest_array)[0]), \
208 					      sizeof((_dest_array)[0])); \
209 		if (!IS_ERR(_tmp)) \
210 			_dest_array = _tmp; \
211 		PTR_ERR_OR_ZERO(_tmp); \
212 	})
213 
214 /**
215  * struct panthor_sync_signal - Represent a synchronization object point to attach
216  * our job fence to.
217  *
218  * This structure is here to keep track of fences that are currently bound to
219  * a specific syncobj point.
220  *
221  * At the beginning of a job submission, the fence
222  * is retrieved from the syncobj itself, and can be NULL if no fence was attached
223  * to this point.
224  *
225  * At the end, it points to the fence of the last job that had a
226  * %DRM_PANTHOR_SYNC_OP_SIGNAL on this syncobj.
227  *
228  * With jobs being submitted in batches, the fence might change several times during
229  * the process, allowing one job to wait on a job that's part of the same submission
230  * but appears earlier in the drm_panthor_group_submit::queue_submits array.
231  */
232 struct panthor_sync_signal {
233 	/** @node: list_head to track signal ops within a submit operation */
234 	struct list_head node;
235 
236 	/** @handle: The syncobj handle. */
237 	u32 handle;
238 
239 	/**
240 	 * @point: The syncobj point.
241 	 *
242 	 * Zero for regular syncobjs, and non-zero for timeline syncobjs.
243 	 */
244 	u64 point;
245 
246 	/**
247 	 * @syncobj: The sync object pointed by @handle.
248 	 */
249 	struct drm_syncobj *syncobj;
250 
251 	/**
252 	 * @chain: Chain object used to link the new fence to an existing
253 	 * timeline syncobj.
254 	 *
255 	 * NULL for regular syncobj, non-NULL for timeline syncobjs.
256 	 */
257 	struct dma_fence_chain *chain;
258 
259 	/**
260 	 * @fence: The fence to assign to the syncobj or syncobj-point.
261 	 */
262 	struct dma_fence *fence;
263 };
264 
265 /**
266  * struct panthor_job_ctx - Job context
267  */
268 struct panthor_job_ctx {
269 	/** @job: The job that is about to be submitted to drm_sched. */
270 	struct drm_sched_job *job;
271 
272 	/** @syncops: Array of sync operations. */
273 	struct drm_panthor_sync_op *syncops;
274 
275 	/** @syncop_count: Number of sync operations. */
276 	u32 syncop_count;
277 };
278 
279 /**
280  * struct panthor_submit_ctx - Submission context
281  *
282  * Anything that's related to a submission (%DRM_IOCTL_PANTHOR_VM_BIND or
283  * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT) is kept here, so we can automate the
284  * initialization and cleanup steps.
285  */
286 struct panthor_submit_ctx {
287 	/** @file: DRM file this submission happens on. */
288 	struct drm_file *file;
289 
290 	/**
291 	 * @signals: List of struct panthor_sync_signal.
292 	 *
293 	 * %DRM_PANTHOR_SYNC_OP_SIGNAL operations will be recorded here,
294 	 * and %DRM_PANTHOR_SYNC_OP_WAIT will first check if an entry
295 	 * matching the syncobj+point exists before calling
296 	 * drm_syncobj_find_fence(). This allows us to describe dependencies
297 	 * existing between jobs that are part of the same batch.
298 	 */
299 	struct list_head signals;
300 
301 	/** @jobs: Array of jobs. */
302 	struct panthor_job_ctx *jobs;
303 
304 	/** @job_count: Number of entries in the @jobs array. */
305 	u32 job_count;
306 
307 	/** @exec: drm_exec context used to acquire and prepare resv objects. */
308 	struct drm_exec exec;
309 };
310 
311 #define PANTHOR_SYNC_OP_FLAGS_MASK \
312 	(DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK | DRM_PANTHOR_SYNC_OP_SIGNAL)
313 
sync_op_is_signal(const struct drm_panthor_sync_op * sync_op)314 static bool sync_op_is_signal(const struct drm_panthor_sync_op *sync_op)
315 {
316 	return !!(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL);
317 }
318 
sync_op_is_wait(const struct drm_panthor_sync_op * sync_op)319 static bool sync_op_is_wait(const struct drm_panthor_sync_op *sync_op)
320 {
321 	/* Note that DRM_PANTHOR_SYNC_OP_WAIT == 0 */
322 	return !(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL);
323 }
324 
325 /**
326  * panthor_check_sync_op() - Check drm_panthor_sync_op fields
327  * @sync_op: The sync operation to check.
328  *
329  * Return: 0 on success, -EINVAL otherwise.
330  */
331 static int
panthor_check_sync_op(const struct drm_panthor_sync_op * sync_op)332 panthor_check_sync_op(const struct drm_panthor_sync_op *sync_op)
333 {
334 	u8 handle_type;
335 
336 	if (sync_op->flags & ~PANTHOR_SYNC_OP_FLAGS_MASK)
337 		return -EINVAL;
338 
339 	handle_type = sync_op->flags & DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK;
340 	if (handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ &&
341 	    handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ)
342 		return -EINVAL;
343 
344 	if (handle_type == DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ &&
345 	    sync_op->timeline_value != 0)
346 		return -EINVAL;
347 
348 	return 0;
349 }
350 
351 /**
352  * panthor_sync_signal_free() - Release resources and free a panthor_sync_signal object
353  * @sig_sync: Signal object to free.
354  */
355 static void
panthor_sync_signal_free(struct panthor_sync_signal * sig_sync)356 panthor_sync_signal_free(struct panthor_sync_signal *sig_sync)
357 {
358 	if (!sig_sync)
359 		return;
360 
361 	drm_syncobj_put(sig_sync->syncobj);
362 	dma_fence_chain_free(sig_sync->chain);
363 	dma_fence_put(sig_sync->fence);
364 	kfree(sig_sync);
365 }
366 
367 /**
368  * panthor_submit_ctx_add_sync_signal() - Add a signal operation to a submit context
369  * @ctx: Context to add the signal operation to.
370  * @handle: Syncobj handle.
371  * @point: Syncobj point.
372  *
373  * Return: 0 on success, otherwise negative error value.
374  */
375 static int
panthor_submit_ctx_add_sync_signal(struct panthor_submit_ctx * ctx,u32 handle,u64 point)376 panthor_submit_ctx_add_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
377 {
378 	struct panthor_sync_signal *sig_sync;
379 	struct dma_fence *cur_fence;
380 	int ret;
381 
382 	sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL);
383 	if (!sig_sync)
384 		return -ENOMEM;
385 
386 	sig_sync->handle = handle;
387 	sig_sync->point = point;
388 
389 	if (point > 0) {
390 		sig_sync->chain = dma_fence_chain_alloc();
391 		if (!sig_sync->chain) {
392 			ret = -ENOMEM;
393 			goto err_free_sig_sync;
394 		}
395 	}
396 
397 	sig_sync->syncobj = drm_syncobj_find(ctx->file, handle);
398 	if (!sig_sync->syncobj) {
399 		ret = -EINVAL;
400 		goto err_free_sig_sync;
401 	}
402 
403 	/* Retrieve the current fence attached to that point. It's
404 	 * perfectly fine to get a NULL fence here, it just means there's
405 	 * no fence attached to that point yet.
406 	 */
407 	if (!drm_syncobj_find_fence(ctx->file, handle, point, 0, &cur_fence))
408 		sig_sync->fence = cur_fence;
409 
410 	list_add_tail(&sig_sync->node, &ctx->signals);
411 
412 	return 0;
413 
414 err_free_sig_sync:
415 	panthor_sync_signal_free(sig_sync);
416 	return ret;
417 }
418 
419 /**
420  * panthor_submit_ctx_search_sync_signal() - Search an existing signal operation in a
421  * submit context.
422  * @ctx: Context to search the signal operation in.
423  * @handle: Syncobj handle.
424  * @point: Syncobj point.
425  *
426  * Return: A valid panthor_sync_signal object if found, NULL otherwise.
427  */
428 static struct panthor_sync_signal *
panthor_submit_ctx_search_sync_signal(struct panthor_submit_ctx * ctx,u32 handle,u64 point)429 panthor_submit_ctx_search_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
430 {
431 	struct panthor_sync_signal *sig_sync;
432 
433 	list_for_each_entry(sig_sync, &ctx->signals, node) {
434 		if (handle == sig_sync->handle && point == sig_sync->point)
435 			return sig_sync;
436 	}
437 
438 	return NULL;
439 }
440 
441 /**
442  * panthor_submit_ctx_add_job() - Add a job to a submit context
443  * @ctx: Context to search the signal operation in.
444  * @idx: Index of the job in the context.
445  * @job: Job to add.
446  * @syncs: Sync operations provided by userspace.
447  *
448  * Return: 0 on success, a negative error code otherwise.
449  */
450 static int
panthor_submit_ctx_add_job(struct panthor_submit_ctx * ctx,u32 idx,struct drm_sched_job * job,const struct drm_panthor_obj_array * syncs)451 panthor_submit_ctx_add_job(struct panthor_submit_ctx *ctx, u32 idx,
452 			   struct drm_sched_job *job,
453 			   const struct drm_panthor_obj_array *syncs)
454 {
455 	int ret;
456 
457 	ctx->jobs[idx].job = job;
458 
459 	ret = PANTHOR_UOBJ_GET_ARRAY(ctx->jobs[idx].syncops, syncs);
460 	if (ret)
461 		return ret;
462 
463 	ctx->jobs[idx].syncop_count = syncs->count;
464 	return 0;
465 }
466 
467 /**
468  * panthor_submit_ctx_get_sync_signal() - Search signal operation and add one if none was found.
469  * @ctx: Context to search the signal operation in.
470  * @handle: Syncobj handle.
471  * @point: Syncobj point.
472  *
473  * Return: 0 on success, a negative error code otherwise.
474  */
475 static int
panthor_submit_ctx_get_sync_signal(struct panthor_submit_ctx * ctx,u32 handle,u64 point)476 panthor_submit_ctx_get_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
477 {
478 	struct panthor_sync_signal *sig_sync;
479 
480 	sig_sync = panthor_submit_ctx_search_sync_signal(ctx, handle, point);
481 	if (sig_sync)
482 		return 0;
483 
484 	return panthor_submit_ctx_add_sync_signal(ctx, handle, point);
485 }
486 
487 /**
488  * panthor_submit_ctx_update_job_sync_signal_fences() - Update fences
489  * on the signal operations specified by a job.
490  * @ctx: Context to search the signal operation in.
491  * @job_idx: Index of the job to operate on.
492  *
493  * Return: 0 on success, a negative error code otherwise.
494  */
495 static int
panthor_submit_ctx_update_job_sync_signal_fences(struct panthor_submit_ctx * ctx,u32 job_idx)496 panthor_submit_ctx_update_job_sync_signal_fences(struct panthor_submit_ctx *ctx,
497 						 u32 job_idx)
498 {
499 	struct panthor_device *ptdev = container_of(ctx->file->minor->dev,
500 						    struct panthor_device,
501 						    base);
502 	struct dma_fence *done_fence = &ctx->jobs[job_idx].job->s_fence->finished;
503 	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
504 	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
505 
506 	for (u32 i = 0; i < sync_op_count; i++) {
507 		struct dma_fence *old_fence;
508 		struct panthor_sync_signal *sig_sync;
509 
510 		if (!sync_op_is_signal(&sync_ops[i]))
511 			continue;
512 
513 		sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle,
514 								 sync_ops[i].timeline_value);
515 		if (drm_WARN_ON(&ptdev->base, !sig_sync))
516 			return -EINVAL;
517 
518 		old_fence = sig_sync->fence;
519 		sig_sync->fence = dma_fence_get(done_fence);
520 		dma_fence_put(old_fence);
521 
522 		if (drm_WARN_ON(&ptdev->base, !sig_sync->fence))
523 			return -EINVAL;
524 	}
525 
526 	return 0;
527 }
528 
529 /**
530  * panthor_submit_ctx_collect_job_signal_ops() - Iterate over all job signal operations
531  * and add them to the context.
532  * @ctx: Context to search the signal operation in.
533  * @job_idx: Index of the job to operate on.
534  *
535  * Return: 0 on success, a negative error code otherwise.
536  */
537 static int
panthor_submit_ctx_collect_job_signal_ops(struct panthor_submit_ctx * ctx,u32 job_idx)538 panthor_submit_ctx_collect_job_signal_ops(struct panthor_submit_ctx *ctx,
539 					  u32 job_idx)
540 {
541 	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
542 	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
543 
544 	for (u32 i = 0; i < sync_op_count; i++) {
545 		int ret;
546 
547 		if (!sync_op_is_signal(&sync_ops[i]))
548 			continue;
549 
550 		ret = panthor_check_sync_op(&sync_ops[i]);
551 		if (ret)
552 			return ret;
553 
554 		ret = panthor_submit_ctx_get_sync_signal(ctx,
555 							 sync_ops[i].handle,
556 							 sync_ops[i].timeline_value);
557 		if (ret)
558 			return ret;
559 	}
560 
561 	return 0;
562 }
563 
564 /**
565  * panthor_submit_ctx_push_fences() - Iterate over the signal array, and for each entry, push
566  * the currently assigned fence to the associated syncobj.
567  * @ctx: Context to push fences on.
568  *
569  * This is the last step of a submission procedure, and is done once we know the submission
570  * is effective and job fences are guaranteed to be signaled in finite time.
571  */
572 static void
panthor_submit_ctx_push_fences(struct panthor_submit_ctx * ctx)573 panthor_submit_ctx_push_fences(struct panthor_submit_ctx *ctx)
574 {
575 	struct panthor_sync_signal *sig_sync;
576 
577 	list_for_each_entry(sig_sync, &ctx->signals, node) {
578 		if (sig_sync->chain) {
579 			drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain,
580 					      sig_sync->fence, sig_sync->point);
581 			sig_sync->chain = NULL;
582 		} else {
583 			drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence);
584 		}
585 	}
586 }
587 
588 /**
589  * panthor_submit_ctx_add_sync_deps_to_job() - Add sync wait operations as
590  * job dependencies.
591  * @ctx: Submit context.
592  * @job_idx: Index of the job to operate on.
593  *
594  * Return: 0 on success, a negative error code otherwise.
595  */
596 static int
panthor_submit_ctx_add_sync_deps_to_job(struct panthor_submit_ctx * ctx,u32 job_idx)597 panthor_submit_ctx_add_sync_deps_to_job(struct panthor_submit_ctx *ctx,
598 					u32 job_idx)
599 {
600 	struct panthor_device *ptdev = container_of(ctx->file->minor->dev,
601 						    struct panthor_device,
602 						    base);
603 	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
604 	struct drm_sched_job *job = ctx->jobs[job_idx].job;
605 	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
606 	int ret = 0;
607 
608 	for (u32 i = 0; i < sync_op_count; i++) {
609 		struct panthor_sync_signal *sig_sync;
610 		struct dma_fence *fence;
611 
612 		if (!sync_op_is_wait(&sync_ops[i]))
613 			continue;
614 
615 		ret = panthor_check_sync_op(&sync_ops[i]);
616 		if (ret)
617 			return ret;
618 
619 		sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle,
620 								 sync_ops[i].timeline_value);
621 		if (sig_sync) {
622 			if (drm_WARN_ON(&ptdev->base, !sig_sync->fence))
623 				return -EINVAL;
624 
625 			fence = dma_fence_get(sig_sync->fence);
626 		} else {
627 			ret = drm_syncobj_find_fence(ctx->file, sync_ops[i].handle,
628 						     sync_ops[i].timeline_value,
629 						     0, &fence);
630 			if (ret)
631 				return ret;
632 		}
633 
634 		ret = drm_sched_job_add_dependency(job, fence);
635 		if (ret)
636 			return ret;
637 	}
638 
639 	return 0;
640 }
641 
642 /**
643  * panthor_submit_ctx_collect_jobs_signal_ops() - Collect all signal operations
644  * and add them to the submit context.
645  * @ctx: Submit context.
646  *
647  * Return: 0 on success, a negative error code otherwise.
648  */
649 static int
panthor_submit_ctx_collect_jobs_signal_ops(struct panthor_submit_ctx * ctx)650 panthor_submit_ctx_collect_jobs_signal_ops(struct panthor_submit_ctx *ctx)
651 {
652 	for (u32 i = 0; i < ctx->job_count; i++) {
653 		int ret;
654 
655 		ret = panthor_submit_ctx_collect_job_signal_ops(ctx, i);
656 		if (ret)
657 			return ret;
658 	}
659 
660 	return 0;
661 }
662 
663 /**
664  * panthor_submit_ctx_add_deps_and_arm_jobs() - Add jobs dependencies and arm jobs
665  * @ctx: Submit context.
666  *
667  * Must be called after the resv preparation has been taken care of.
668  *
669  * Return: 0 on success, a negative error code otherwise.
670  */
671 static int
panthor_submit_ctx_add_deps_and_arm_jobs(struct panthor_submit_ctx * ctx)672 panthor_submit_ctx_add_deps_and_arm_jobs(struct panthor_submit_ctx *ctx)
673 {
674 	for (u32 i = 0; i < ctx->job_count; i++) {
675 		int ret;
676 
677 		ret = panthor_submit_ctx_add_sync_deps_to_job(ctx, i);
678 		if (ret)
679 			return ret;
680 
681 		drm_sched_job_arm(ctx->jobs[i].job);
682 
683 		ret = panthor_submit_ctx_update_job_sync_signal_fences(ctx, i);
684 		if (ret)
685 			return ret;
686 	}
687 
688 	return 0;
689 }
690 
691 /**
692  * panthor_submit_ctx_push_jobs() - Push jobs to their scheduling entities.
693  * @ctx: Submit context.
694  * @upd_resvs: Callback used to update reservation objects that were previously
695  * preapred.
696  */
697 static void
panthor_submit_ctx_push_jobs(struct panthor_submit_ctx * ctx,void (* upd_resvs)(struct drm_exec *,struct drm_sched_job *))698 panthor_submit_ctx_push_jobs(struct panthor_submit_ctx *ctx,
699 			     void (*upd_resvs)(struct drm_exec *, struct drm_sched_job *))
700 {
701 	for (u32 i = 0; i < ctx->job_count; i++) {
702 		upd_resvs(&ctx->exec, ctx->jobs[i].job);
703 		drm_sched_entity_push_job(ctx->jobs[i].job);
704 
705 		/* Job is owned by the scheduler now. */
706 		ctx->jobs[i].job = NULL;
707 	}
708 
709 	panthor_submit_ctx_push_fences(ctx);
710 }
711 
712 /**
713  * panthor_submit_ctx_init() - Initializes a submission context
714  * @ctx: Submit context to initialize.
715  * @file: drm_file this submission happens on.
716  * @job_count: Number of jobs that will be submitted.
717  *
718  * Return: 0 on success, a negative error code otherwise.
719  */
panthor_submit_ctx_init(struct panthor_submit_ctx * ctx,struct drm_file * file,u32 job_count)720 static int panthor_submit_ctx_init(struct panthor_submit_ctx *ctx,
721 				   struct drm_file *file, u32 job_count)
722 {
723 	ctx->jobs = kvmalloc_array(job_count, sizeof(*ctx->jobs),
724 				   GFP_KERNEL | __GFP_ZERO);
725 	if (!ctx->jobs)
726 		return -ENOMEM;
727 
728 	ctx->file = file;
729 	ctx->job_count = job_count;
730 	INIT_LIST_HEAD(&ctx->signals);
731 	drm_exec_init(&ctx->exec,
732 		      DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
733 		      0);
734 	return 0;
735 }
736 
737 /**
738  * panthor_submit_ctx_cleanup() - Cleanup a submission context
739  * @ctx: Submit context to cleanup.
740  * @job_put: Job put callback.
741  */
panthor_submit_ctx_cleanup(struct panthor_submit_ctx * ctx,void (* job_put)(struct drm_sched_job *))742 static void panthor_submit_ctx_cleanup(struct panthor_submit_ctx *ctx,
743 				       void (*job_put)(struct drm_sched_job *))
744 {
745 	struct panthor_sync_signal *sig_sync, *tmp;
746 	unsigned long i;
747 
748 	drm_exec_fini(&ctx->exec);
749 
750 	list_for_each_entry_safe(sig_sync, tmp, &ctx->signals, node)
751 		panthor_sync_signal_free(sig_sync);
752 
753 	for (i = 0; i < ctx->job_count; i++) {
754 		job_put(ctx->jobs[i].job);
755 		kvfree(ctx->jobs[i].syncops);
756 	}
757 
758 	kvfree(ctx->jobs);
759 }
760 
panthor_query_timestamp_info(struct panthor_device * ptdev,struct drm_panthor_timestamp_info * arg)761 static int panthor_query_timestamp_info(struct panthor_device *ptdev,
762 					struct drm_panthor_timestamp_info *arg)
763 {
764 	int ret;
765 
766 	ret = panthor_device_resume_and_get(ptdev);
767 	if (ret)
768 		return ret;
769 
770 #ifdef CONFIG_ARM_ARCH_TIMER
771 	arg->timestamp_frequency = arch_timer_get_cntfrq();
772 #else
773 	arg->timestamp_frequency = 0;
774 #endif
775 	arg->current_timestamp = panthor_gpu_read_timestamp(ptdev);
776 	arg->timestamp_offset = panthor_gpu_read_timestamp_offset(ptdev);
777 
778 	pm_runtime_put(ptdev->base.dev);
779 	return 0;
780 }
781 
group_priority_permit(struct drm_file * file,u8 priority)782 static int group_priority_permit(struct drm_file *file,
783 				 u8 priority)
784 {
785 	/* Ensure that priority is valid */
786 	if (priority > PANTHOR_GROUP_PRIORITY_REALTIME)
787 		return -EINVAL;
788 
789 	/* Medium priority and below are always allowed */
790 	if (priority <= PANTHOR_GROUP_PRIORITY_MEDIUM)
791 		return 0;
792 
793 	/* Higher priorities require CAP_SYS_NICE or DRM_MASTER */
794 	if (capable(CAP_SYS_NICE) || drm_is_current_master(file))
795 		return 0;
796 
797 	return -EACCES;
798 }
799 
panthor_query_group_priorities_info(struct drm_file * file,struct drm_panthor_group_priorities_info * arg)800 static void panthor_query_group_priorities_info(struct drm_file *file,
801 						struct drm_panthor_group_priorities_info *arg)
802 {
803 	int prio;
804 
805 	memset(arg, 0, sizeof(*arg));
806 	for (prio = PANTHOR_GROUP_PRIORITY_REALTIME; prio >= 0; prio--) {
807 		if (!group_priority_permit(file, prio))
808 			arg->allowed_mask |= BIT(prio);
809 	}
810 }
811 
panthor_ioctl_dev_query(struct drm_device * ddev,void * data,struct drm_file * file)812 static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct drm_file *file)
813 {
814 	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
815 	struct drm_panthor_dev_query *args = data;
816 	struct drm_panthor_timestamp_info timestamp_info;
817 	struct drm_panthor_group_priorities_info priorities_info;
818 	int ret;
819 
820 	if (!args->pointer) {
821 		switch (args->type) {
822 		case DRM_PANTHOR_DEV_QUERY_GPU_INFO:
823 			args->size = sizeof(ptdev->gpu_info);
824 			return 0;
825 
826 		case DRM_PANTHOR_DEV_QUERY_CSIF_INFO:
827 			args->size = sizeof(ptdev->csif_info);
828 			return 0;
829 
830 		case DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO:
831 			args->size = sizeof(timestamp_info);
832 			return 0;
833 
834 		case DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO:
835 			args->size = sizeof(priorities_info);
836 			return 0;
837 
838 		default:
839 			return -EINVAL;
840 		}
841 	}
842 
843 	switch (args->type) {
844 	case DRM_PANTHOR_DEV_QUERY_GPU_INFO:
845 		return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->gpu_info);
846 
847 	case DRM_PANTHOR_DEV_QUERY_CSIF_INFO:
848 		return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->csif_info);
849 
850 	case DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO:
851 		ret = panthor_query_timestamp_info(ptdev, &timestamp_info);
852 
853 		if (ret)
854 			return ret;
855 
856 		return PANTHOR_UOBJ_SET(args->pointer, args->size, timestamp_info);
857 
858 	case DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO:
859 		panthor_query_group_priorities_info(file, &priorities_info);
860 		return PANTHOR_UOBJ_SET(args->pointer, args->size, priorities_info);
861 
862 	default:
863 		return -EINVAL;
864 	}
865 }
866 
867 #define PANTHOR_VM_CREATE_FLAGS			0
868 
panthor_ioctl_vm_create(struct drm_device * ddev,void * data,struct drm_file * file)869 static int panthor_ioctl_vm_create(struct drm_device *ddev, void *data,
870 				   struct drm_file *file)
871 {
872 	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
873 	struct panthor_file *pfile = file->driver_priv;
874 	struct drm_panthor_vm_create *args = data;
875 	int cookie, ret;
876 
877 	if (!drm_dev_enter(ddev, &cookie))
878 		return -ENODEV;
879 
880 	ret = panthor_vm_pool_create_vm(ptdev, pfile->vms,  args);
881 	if (ret >= 0) {
882 		args->id = ret;
883 		ret = 0;
884 	}
885 
886 	drm_dev_exit(cookie);
887 	return ret;
888 }
889 
panthor_ioctl_vm_destroy(struct drm_device * ddev,void * data,struct drm_file * file)890 static int panthor_ioctl_vm_destroy(struct drm_device *ddev, void *data,
891 				    struct drm_file *file)
892 {
893 	struct panthor_file *pfile = file->driver_priv;
894 	struct drm_panthor_vm_destroy *args = data;
895 
896 	if (args->pad)
897 		return -EINVAL;
898 
899 	return panthor_vm_pool_destroy_vm(pfile->vms, args->id);
900 }
901 
902 #define PANTHOR_BO_FLAGS		DRM_PANTHOR_BO_NO_MMAP
903 
panthor_ioctl_bo_create(struct drm_device * ddev,void * data,struct drm_file * file)904 static int panthor_ioctl_bo_create(struct drm_device *ddev, void *data,
905 				   struct drm_file *file)
906 {
907 	struct panthor_file *pfile = file->driver_priv;
908 	struct drm_panthor_bo_create *args = data;
909 	struct panthor_vm *vm = NULL;
910 	int cookie, ret;
911 
912 	if (!drm_dev_enter(ddev, &cookie))
913 		return -ENODEV;
914 
915 	if (!args->size || args->pad ||
916 	    (args->flags & ~PANTHOR_BO_FLAGS)) {
917 		ret = -EINVAL;
918 		goto out_dev_exit;
919 	}
920 
921 	if (args->exclusive_vm_id) {
922 		vm = panthor_vm_pool_get_vm(pfile->vms, args->exclusive_vm_id);
923 		if (!vm) {
924 			ret = -EINVAL;
925 			goto out_dev_exit;
926 		}
927 	}
928 
929 	ret = panthor_gem_create_with_handle(file, ddev, vm, &args->size,
930 					     args->flags, &args->handle);
931 
932 	panthor_vm_put(vm);
933 
934 out_dev_exit:
935 	drm_dev_exit(cookie);
936 	return ret;
937 }
938 
panthor_ioctl_bo_mmap_offset(struct drm_device * ddev,void * data,struct drm_file * file)939 static int panthor_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data,
940 					struct drm_file *file)
941 {
942 	struct drm_panthor_bo_mmap_offset *args = data;
943 	struct drm_gem_object *obj;
944 	int ret;
945 
946 	if (args->pad)
947 		return -EINVAL;
948 
949 	obj = drm_gem_object_lookup(file, args->handle);
950 	if (!obj)
951 		return -ENOENT;
952 
953 	ret = drm_gem_create_mmap_offset(obj);
954 	if (ret)
955 		goto out;
956 
957 	args->offset = drm_vma_node_offset_addr(&obj->vma_node);
958 
959 out:
960 	drm_gem_object_put(obj);
961 	return ret;
962 }
963 
panthor_ioctl_group_submit(struct drm_device * ddev,void * data,struct drm_file * file)964 static int panthor_ioctl_group_submit(struct drm_device *ddev, void *data,
965 				      struct drm_file *file)
966 {
967 	struct panthor_file *pfile = file->driver_priv;
968 	struct drm_panthor_group_submit *args = data;
969 	struct drm_panthor_queue_submit *jobs_args;
970 	struct panthor_submit_ctx ctx;
971 	int ret = 0, cookie;
972 
973 	if (args->pad)
974 		return -EINVAL;
975 
976 	if (!drm_dev_enter(ddev, &cookie))
977 		return -ENODEV;
978 
979 	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->queue_submits);
980 	if (ret)
981 		goto out_dev_exit;
982 
983 	ret = panthor_submit_ctx_init(&ctx, file, args->queue_submits.count);
984 	if (ret)
985 		goto out_free_jobs_args;
986 
987 	/* Create jobs and attach sync operations */
988 	for (u32 i = 0; i < args->queue_submits.count; i++) {
989 		const struct drm_panthor_queue_submit *qsubmit = &jobs_args[i];
990 		struct drm_sched_job *job;
991 
992 		job = panthor_job_create(pfile, args->group_handle, qsubmit);
993 		if (IS_ERR(job)) {
994 			ret = PTR_ERR(job);
995 			goto out_cleanup_submit_ctx;
996 		}
997 
998 		ret = panthor_submit_ctx_add_job(&ctx, i, job, &qsubmit->syncs);
999 		if (ret)
1000 			goto out_cleanup_submit_ctx;
1001 	}
1002 
1003 	/*
1004 	 * Collect signal operations on all jobs, such that each job can pick
1005 	 * from it for its dependencies and update the fence to signal when the
1006 	 * job is submitted.
1007 	 */
1008 	ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx);
1009 	if (ret)
1010 		goto out_cleanup_submit_ctx;
1011 
1012 	/*
1013 	 * We acquire/prepare revs on all jobs before proceeding with the
1014 	 * dependency registration.
1015 	 *
1016 	 * This is solving two problems:
1017 	 * 1. drm_sched_job_arm() and drm_sched_entity_push_job() must be
1018 	 *    protected by a lock to make sure no concurrent access to the same
1019 	 *    entity get interleaved, which would mess up with the fence seqno
1020 	 *    ordering. Luckily, one of the resv being acquired is the VM resv,
1021 	 *    and a scheduling entity is only bound to a single VM. As soon as
1022 	 *    we acquire the VM resv, we should be safe.
1023 	 * 2. Jobs might depend on fences that were issued by previous jobs in
1024 	 *    the same batch, so we can't add dependencies on all jobs before
1025 	 *    arming previous jobs and registering the fence to the signal
1026 	 *    array, otherwise we might miss dependencies, or point to an
1027 	 *    outdated fence.
1028 	 */
1029 	if (args->queue_submits.count > 0) {
1030 		/* All jobs target the same group, so they also point to the same VM. */
1031 		struct panthor_vm *vm = panthor_job_vm(ctx.jobs[0].job);
1032 
1033 		drm_exec_until_all_locked(&ctx.exec) {
1034 			ret = panthor_vm_prepare_mapped_bos_resvs(&ctx.exec, vm,
1035 								  args->queue_submits.count);
1036 		}
1037 
1038 		if (ret)
1039 			goto out_cleanup_submit_ctx;
1040 	}
1041 
1042 	/*
1043 	 * Now that resvs are locked/prepared, we can iterate over each job to
1044 	 * add the dependencies, arm the job fence, register the job fence to
1045 	 * the signal array.
1046 	 */
1047 	ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx);
1048 	if (ret)
1049 		goto out_cleanup_submit_ctx;
1050 
1051 	/* Nothing can fail after that point, so we can make our job fences
1052 	 * visible to the outside world. Push jobs and set the job fences to
1053 	 * the resv slots we reserved.  This also pushes the fences to the
1054 	 * syncobjs that are part of the signal array.
1055 	 */
1056 	panthor_submit_ctx_push_jobs(&ctx, panthor_job_update_resvs);
1057 
1058 out_cleanup_submit_ctx:
1059 	panthor_submit_ctx_cleanup(&ctx, panthor_job_put);
1060 
1061 out_free_jobs_args:
1062 	kvfree(jobs_args);
1063 
1064 out_dev_exit:
1065 	drm_dev_exit(cookie);
1066 	return ret;
1067 }
1068 
panthor_ioctl_group_destroy(struct drm_device * ddev,void * data,struct drm_file * file)1069 static int panthor_ioctl_group_destroy(struct drm_device *ddev, void *data,
1070 				       struct drm_file *file)
1071 {
1072 	struct panthor_file *pfile = file->driver_priv;
1073 	struct drm_panthor_group_destroy *args = data;
1074 
1075 	if (args->pad)
1076 		return -EINVAL;
1077 
1078 	return panthor_group_destroy(pfile, args->group_handle);
1079 }
1080 
panthor_ioctl_group_create(struct drm_device * ddev,void * data,struct drm_file * file)1081 static int panthor_ioctl_group_create(struct drm_device *ddev, void *data,
1082 				      struct drm_file *file)
1083 {
1084 	struct panthor_file *pfile = file->driver_priv;
1085 	struct drm_panthor_group_create *args = data;
1086 	struct drm_panthor_queue_create *queue_args;
1087 	int ret;
1088 
1089 	if (!args->queues.count)
1090 		return -EINVAL;
1091 
1092 	ret = PANTHOR_UOBJ_GET_ARRAY(queue_args, &args->queues);
1093 	if (ret)
1094 		return ret;
1095 
1096 	ret = group_priority_permit(file, args->priority);
1097 	if (ret)
1098 		return ret;
1099 
1100 	ret = panthor_group_create(pfile, args, queue_args);
1101 	if (ret >= 0) {
1102 		args->group_handle = ret;
1103 		ret = 0;
1104 	}
1105 
1106 	kvfree(queue_args);
1107 	return ret;
1108 }
1109 
panthor_ioctl_group_get_state(struct drm_device * ddev,void * data,struct drm_file * file)1110 static int panthor_ioctl_group_get_state(struct drm_device *ddev, void *data,
1111 					 struct drm_file *file)
1112 {
1113 	struct panthor_file *pfile = file->driver_priv;
1114 	struct drm_panthor_group_get_state *args = data;
1115 
1116 	return panthor_group_get_state(pfile, args);
1117 }
1118 
panthor_ioctl_tiler_heap_create(struct drm_device * ddev,void * data,struct drm_file * file)1119 static int panthor_ioctl_tiler_heap_create(struct drm_device *ddev, void *data,
1120 					   struct drm_file *file)
1121 {
1122 	struct panthor_file *pfile = file->driver_priv;
1123 	struct drm_panthor_tiler_heap_create *args = data;
1124 	struct panthor_heap_pool *pool;
1125 	struct panthor_vm *vm;
1126 	int ret;
1127 
1128 	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
1129 	if (!vm)
1130 		return -EINVAL;
1131 
1132 	pool = panthor_vm_get_heap_pool(vm, true);
1133 	if (IS_ERR(pool)) {
1134 		ret = PTR_ERR(pool);
1135 		goto out_put_vm;
1136 	}
1137 
1138 	ret = panthor_heap_create(pool,
1139 				  args->initial_chunk_count,
1140 				  args->chunk_size,
1141 				  args->max_chunks,
1142 				  args->target_in_flight,
1143 				  &args->tiler_heap_ctx_gpu_va,
1144 				  &args->first_heap_chunk_gpu_va);
1145 	if (ret < 0)
1146 		goto out_put_heap_pool;
1147 
1148 	/* Heap pools are per-VM. We combine the VM and HEAP id to make
1149 	 * a unique heap handle.
1150 	 */
1151 	args->handle = (args->vm_id << 16) | ret;
1152 	ret = 0;
1153 
1154 out_put_heap_pool:
1155 	panthor_heap_pool_put(pool);
1156 
1157 out_put_vm:
1158 	panthor_vm_put(vm);
1159 	return ret;
1160 }
1161 
panthor_ioctl_tiler_heap_destroy(struct drm_device * ddev,void * data,struct drm_file * file)1162 static int panthor_ioctl_tiler_heap_destroy(struct drm_device *ddev, void *data,
1163 					    struct drm_file *file)
1164 {
1165 	struct panthor_file *pfile = file->driver_priv;
1166 	struct drm_panthor_tiler_heap_destroy *args = data;
1167 	struct panthor_heap_pool *pool;
1168 	struct panthor_vm *vm;
1169 	int ret;
1170 
1171 	if (args->pad)
1172 		return -EINVAL;
1173 
1174 	vm = panthor_vm_pool_get_vm(pfile->vms, args->handle >> 16);
1175 	if (!vm)
1176 		return -EINVAL;
1177 
1178 	pool = panthor_vm_get_heap_pool(vm, false);
1179 	if (IS_ERR(pool)) {
1180 		ret = PTR_ERR(pool);
1181 		goto out_put_vm;
1182 	}
1183 
1184 	ret = panthor_heap_destroy(pool, args->handle & GENMASK(15, 0));
1185 	panthor_heap_pool_put(pool);
1186 
1187 out_put_vm:
1188 	panthor_vm_put(vm);
1189 	return ret;
1190 }
1191 
panthor_ioctl_vm_bind_async(struct drm_device * ddev,struct drm_panthor_vm_bind * args,struct drm_file * file)1192 static int panthor_ioctl_vm_bind_async(struct drm_device *ddev,
1193 				       struct drm_panthor_vm_bind *args,
1194 				       struct drm_file *file)
1195 {
1196 	struct panthor_file *pfile = file->driver_priv;
1197 	struct drm_panthor_vm_bind_op *jobs_args;
1198 	struct panthor_submit_ctx ctx;
1199 	struct panthor_vm *vm;
1200 	int ret = 0;
1201 
1202 	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
1203 	if (!vm)
1204 		return -EINVAL;
1205 
1206 	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops);
1207 	if (ret)
1208 		goto out_put_vm;
1209 
1210 	ret = panthor_submit_ctx_init(&ctx, file, args->ops.count);
1211 	if (ret)
1212 		goto out_free_jobs_args;
1213 
1214 	for (u32 i = 0; i < args->ops.count; i++) {
1215 		struct drm_panthor_vm_bind_op *op = &jobs_args[i];
1216 		struct drm_sched_job *job;
1217 
1218 		job = panthor_vm_bind_job_create(file, vm, op);
1219 		if (IS_ERR(job)) {
1220 			ret = PTR_ERR(job);
1221 			goto out_cleanup_submit_ctx;
1222 		}
1223 
1224 		ret = panthor_submit_ctx_add_job(&ctx, i, job, &op->syncs);
1225 		if (ret)
1226 			goto out_cleanup_submit_ctx;
1227 	}
1228 
1229 	ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx);
1230 	if (ret)
1231 		goto out_cleanup_submit_ctx;
1232 
1233 	/* Prepare reservation objects for each VM_BIND job. */
1234 	drm_exec_until_all_locked(&ctx.exec) {
1235 		for (u32 i = 0; i < ctx.job_count; i++) {
1236 			ret = panthor_vm_bind_job_prepare_resvs(&ctx.exec, ctx.jobs[i].job);
1237 			drm_exec_retry_on_contention(&ctx.exec);
1238 			if (ret)
1239 				goto out_cleanup_submit_ctx;
1240 		}
1241 	}
1242 
1243 	ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx);
1244 	if (ret)
1245 		goto out_cleanup_submit_ctx;
1246 
1247 	/* Nothing can fail after that point. */
1248 	panthor_submit_ctx_push_jobs(&ctx, panthor_vm_bind_job_update_resvs);
1249 
1250 out_cleanup_submit_ctx:
1251 	panthor_submit_ctx_cleanup(&ctx, panthor_vm_bind_job_put);
1252 
1253 out_free_jobs_args:
1254 	kvfree(jobs_args);
1255 
1256 out_put_vm:
1257 	panthor_vm_put(vm);
1258 	return ret;
1259 }
1260 
panthor_ioctl_vm_bind_sync(struct drm_device * ddev,struct drm_panthor_vm_bind * args,struct drm_file * file)1261 static int panthor_ioctl_vm_bind_sync(struct drm_device *ddev,
1262 				      struct drm_panthor_vm_bind *args,
1263 				      struct drm_file *file)
1264 {
1265 	struct panthor_file *pfile = file->driver_priv;
1266 	struct drm_panthor_vm_bind_op *jobs_args;
1267 	struct panthor_vm *vm;
1268 	int ret;
1269 
1270 	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
1271 	if (!vm)
1272 		return -EINVAL;
1273 
1274 	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops);
1275 	if (ret)
1276 		goto out_put_vm;
1277 
1278 	for (u32 i = 0; i < args->ops.count; i++) {
1279 		ret = panthor_vm_bind_exec_sync_op(file, vm, &jobs_args[i]);
1280 		if (ret) {
1281 			/* Update ops.count so the user knows where things failed. */
1282 			args->ops.count = i;
1283 			break;
1284 		}
1285 	}
1286 
1287 	kvfree(jobs_args);
1288 
1289 out_put_vm:
1290 	panthor_vm_put(vm);
1291 	return ret;
1292 }
1293 
1294 #define PANTHOR_VM_BIND_FLAGS DRM_PANTHOR_VM_BIND_ASYNC
1295 
panthor_ioctl_vm_bind(struct drm_device * ddev,void * data,struct drm_file * file)1296 static int panthor_ioctl_vm_bind(struct drm_device *ddev, void *data,
1297 				 struct drm_file *file)
1298 {
1299 	struct drm_panthor_vm_bind *args = data;
1300 	int cookie, ret;
1301 
1302 	if (!drm_dev_enter(ddev, &cookie))
1303 		return -ENODEV;
1304 
1305 	if (args->flags & DRM_PANTHOR_VM_BIND_ASYNC)
1306 		ret = panthor_ioctl_vm_bind_async(ddev, args, file);
1307 	else
1308 		ret = panthor_ioctl_vm_bind_sync(ddev, args, file);
1309 
1310 	drm_dev_exit(cookie);
1311 	return ret;
1312 }
1313 
panthor_ioctl_vm_get_state(struct drm_device * ddev,void * data,struct drm_file * file)1314 static int panthor_ioctl_vm_get_state(struct drm_device *ddev, void *data,
1315 				      struct drm_file *file)
1316 {
1317 	struct panthor_file *pfile = file->driver_priv;
1318 	struct drm_panthor_vm_get_state *args = data;
1319 	struct panthor_vm *vm;
1320 
1321 	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
1322 	if (!vm)
1323 		return -EINVAL;
1324 
1325 	if (panthor_vm_is_unusable(vm))
1326 		args->state = DRM_PANTHOR_VM_STATE_UNUSABLE;
1327 	else
1328 		args->state = DRM_PANTHOR_VM_STATE_USABLE;
1329 
1330 	panthor_vm_put(vm);
1331 	return 0;
1332 }
1333 
1334 static int
panthor_open(struct drm_device * ddev,struct drm_file * file)1335 panthor_open(struct drm_device *ddev, struct drm_file *file)
1336 {
1337 	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
1338 	struct panthor_file *pfile;
1339 	int ret;
1340 
1341 	if (!try_module_get(THIS_MODULE))
1342 		return -EINVAL;
1343 
1344 	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
1345 	if (!pfile) {
1346 		ret = -ENOMEM;
1347 		goto err_put_mod;
1348 	}
1349 
1350 	pfile->ptdev = ptdev;
1351 
1352 	ret = panthor_vm_pool_create(pfile);
1353 	if (ret)
1354 		goto err_free_file;
1355 
1356 	ret = panthor_group_pool_create(pfile);
1357 	if (ret)
1358 		goto err_destroy_vm_pool;
1359 
1360 	file->driver_priv = pfile;
1361 	return 0;
1362 
1363 err_destroy_vm_pool:
1364 	panthor_vm_pool_destroy(pfile);
1365 
1366 err_free_file:
1367 	kfree(pfile);
1368 
1369 err_put_mod:
1370 	module_put(THIS_MODULE);
1371 	return ret;
1372 }
1373 
1374 static void
panthor_postclose(struct drm_device * ddev,struct drm_file * file)1375 panthor_postclose(struct drm_device *ddev, struct drm_file *file)
1376 {
1377 	struct panthor_file *pfile = file->driver_priv;
1378 
1379 	panthor_group_pool_destroy(pfile);
1380 	panthor_vm_pool_destroy(pfile);
1381 
1382 	kfree(pfile);
1383 	module_put(THIS_MODULE);
1384 }
1385 
1386 static const struct drm_ioctl_desc panthor_drm_driver_ioctls[] = {
1387 #define PANTHOR_IOCTL(n, func, flags) \
1388 	DRM_IOCTL_DEF_DRV(PANTHOR_##n, panthor_ioctl_##func, flags)
1389 
1390 	PANTHOR_IOCTL(DEV_QUERY, dev_query, DRM_RENDER_ALLOW),
1391 	PANTHOR_IOCTL(VM_CREATE, vm_create, DRM_RENDER_ALLOW),
1392 	PANTHOR_IOCTL(VM_DESTROY, vm_destroy, DRM_RENDER_ALLOW),
1393 	PANTHOR_IOCTL(VM_BIND, vm_bind, DRM_RENDER_ALLOW),
1394 	PANTHOR_IOCTL(VM_GET_STATE, vm_get_state, DRM_RENDER_ALLOW),
1395 	PANTHOR_IOCTL(BO_CREATE, bo_create, DRM_RENDER_ALLOW),
1396 	PANTHOR_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, DRM_RENDER_ALLOW),
1397 	PANTHOR_IOCTL(GROUP_CREATE, group_create, DRM_RENDER_ALLOW),
1398 	PANTHOR_IOCTL(GROUP_DESTROY, group_destroy, DRM_RENDER_ALLOW),
1399 	PANTHOR_IOCTL(GROUP_GET_STATE, group_get_state, DRM_RENDER_ALLOW),
1400 	PANTHOR_IOCTL(TILER_HEAP_CREATE, tiler_heap_create, DRM_RENDER_ALLOW),
1401 	PANTHOR_IOCTL(TILER_HEAP_DESTROY, tiler_heap_destroy, DRM_RENDER_ALLOW),
1402 	PANTHOR_IOCTL(GROUP_SUBMIT, group_submit, DRM_RENDER_ALLOW),
1403 };
1404 
panthor_mmap(struct file * filp,struct vm_area_struct * vma)1405 static int panthor_mmap(struct file *filp, struct vm_area_struct *vma)
1406 {
1407 	struct drm_file *file = filp->private_data;
1408 	struct panthor_file *pfile = file->driver_priv;
1409 	struct panthor_device *ptdev = pfile->ptdev;
1410 	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
1411 	int ret, cookie;
1412 
1413 	if (!drm_dev_enter(file->minor->dev, &cookie))
1414 		return -ENODEV;
1415 
1416 #ifdef CONFIG_ARM64
1417 	/*
1418 	 * With 32-bit systems being limited by the 32-bit representation of
1419 	 * mmap2's pgoffset field, we need to make the MMIO offset arch
1420 	 * specific. This converts a user MMIO offset into something the kernel
1421 	 * driver understands.
1422 	 */
1423 	if (test_tsk_thread_flag(current, TIF_32BIT) &&
1424 	    offset >= DRM_PANTHOR_USER_MMIO_OFFSET_32BIT) {
1425 		offset += DRM_PANTHOR_USER_MMIO_OFFSET_64BIT -
1426 			  DRM_PANTHOR_USER_MMIO_OFFSET_32BIT;
1427 		vma->vm_pgoff = offset >> PAGE_SHIFT;
1428 	}
1429 #endif
1430 
1431 	if (offset >= DRM_PANTHOR_USER_MMIO_OFFSET)
1432 		ret = panthor_device_mmap_io(ptdev, vma);
1433 	else
1434 		ret = drm_gem_mmap(filp, vma);
1435 
1436 	drm_dev_exit(cookie);
1437 	return ret;
1438 }
1439 
panthor_gpu_show_fdinfo(struct panthor_device * ptdev,struct panthor_file * pfile,struct drm_printer * p)1440 static void panthor_gpu_show_fdinfo(struct panthor_device *ptdev,
1441 				    struct panthor_file *pfile,
1442 				    struct drm_printer *p)
1443 {
1444 	if (ptdev->profile_mask & PANTHOR_DEVICE_PROFILING_ALL)
1445 		panthor_fdinfo_gather_group_samples(pfile);
1446 
1447 	if (ptdev->profile_mask & PANTHOR_DEVICE_PROFILING_TIMESTAMP) {
1448 #ifdef CONFIG_ARM_ARCH_TIMER
1449 		drm_printf(p, "drm-engine-panthor:\t%llu ns\n",
1450 			   DIV_ROUND_UP_ULL((pfile->stats.time * NSEC_PER_SEC),
1451 					    arch_timer_get_cntfrq()));
1452 #endif
1453 	}
1454 	if (ptdev->profile_mask & PANTHOR_DEVICE_PROFILING_CYCLES)
1455 		drm_printf(p, "drm-cycles-panthor:\t%llu\n", pfile->stats.cycles);
1456 
1457 	drm_printf(p, "drm-maxfreq-panthor:\t%lu Hz\n", ptdev->fast_rate);
1458 	drm_printf(p, "drm-curfreq-panthor:\t%lu Hz\n", ptdev->current_frequency);
1459 }
1460 
panthor_show_fdinfo(struct drm_printer * p,struct drm_file * file)1461 static void panthor_show_fdinfo(struct drm_printer *p, struct drm_file *file)
1462 {
1463 	struct drm_device *dev = file->minor->dev;
1464 	struct panthor_device *ptdev = container_of(dev, struct panthor_device, base);
1465 
1466 	panthor_gpu_show_fdinfo(ptdev, file->driver_priv, p);
1467 
1468 	drm_show_memory_stats(p, file);
1469 }
1470 
1471 static const struct file_operations panthor_drm_driver_fops = {
1472 	.open = drm_open,
1473 	.release = drm_release,
1474 	.unlocked_ioctl = drm_ioctl,
1475 	.compat_ioctl = drm_compat_ioctl,
1476 	.poll = drm_poll,
1477 	.read = drm_read,
1478 	.llseek = noop_llseek,
1479 	.mmap = panthor_mmap,
1480 	.show_fdinfo = drm_show_fdinfo,
1481 	.fop_flags = FOP_UNSIGNED_OFFSET,
1482 };
1483 
1484 #ifdef CONFIG_DEBUG_FS
panthor_debugfs_init(struct drm_minor * minor)1485 static void panthor_debugfs_init(struct drm_minor *minor)
1486 {
1487 	panthor_mmu_debugfs_init(minor);
1488 }
1489 #endif
1490 
1491 /*
1492  * PanCSF driver version:
1493  * - 1.0 - initial interface
1494  * - 1.1 - adds DEV_QUERY_TIMESTAMP_INFO query
1495  * - 1.2 - adds DEV_QUERY_GROUP_PRIORITIES_INFO query
1496  *       - adds PANTHOR_GROUP_PRIORITY_REALTIME priority
1497  * - 1.3 - adds DRM_PANTHOR_GROUP_STATE_INNOCENT flag
1498  */
1499 static const struct drm_driver panthor_drm_driver = {
1500 	.driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ |
1501 			   DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
1502 	.open = panthor_open,
1503 	.postclose = panthor_postclose,
1504 	.show_fdinfo = panthor_show_fdinfo,
1505 	.ioctls = panthor_drm_driver_ioctls,
1506 	.num_ioctls = ARRAY_SIZE(panthor_drm_driver_ioctls),
1507 	.fops = &panthor_drm_driver_fops,
1508 	.name = "panthor",
1509 	.desc = "Panthor DRM driver",
1510 	.major = 1,
1511 	.minor = 3,
1512 
1513 	.gem_create_object = panthor_gem_create_object,
1514 	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
1515 #ifdef CONFIG_DEBUG_FS
1516 	.debugfs_init = panthor_debugfs_init,
1517 #endif
1518 };
1519 
panthor_probe(struct platform_device * pdev)1520 static int panthor_probe(struct platform_device *pdev)
1521 {
1522 	struct panthor_device *ptdev;
1523 
1524 	ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver,
1525 				   struct panthor_device, base);
1526 	if (IS_ERR(ptdev))
1527 		return -ENOMEM;
1528 
1529 	platform_set_drvdata(pdev, ptdev);
1530 
1531 	return panthor_device_init(ptdev);
1532 }
1533 
panthor_remove(struct platform_device * pdev)1534 static void panthor_remove(struct platform_device *pdev)
1535 {
1536 	struct panthor_device *ptdev = platform_get_drvdata(pdev);
1537 
1538 	panthor_device_unplug(ptdev);
1539 }
1540 
profiling_show(struct device * dev,struct device_attribute * attr,char * buf)1541 static ssize_t profiling_show(struct device *dev,
1542 			      struct device_attribute *attr,
1543 			      char *buf)
1544 {
1545 	struct panthor_device *ptdev = dev_get_drvdata(dev);
1546 
1547 	return sysfs_emit(buf, "%d\n", ptdev->profile_mask);
1548 }
1549 
profiling_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)1550 static ssize_t profiling_store(struct device *dev,
1551 			       struct device_attribute *attr,
1552 			       const char *buf, size_t len)
1553 {
1554 	struct panthor_device *ptdev = dev_get_drvdata(dev);
1555 	u32 value;
1556 	int err;
1557 
1558 	err = kstrtou32(buf, 0, &value);
1559 	if (err)
1560 		return err;
1561 
1562 	if ((value & ~PANTHOR_DEVICE_PROFILING_ALL) != 0)
1563 		return -EINVAL;
1564 
1565 	ptdev->profile_mask = value;
1566 
1567 	return len;
1568 }
1569 
1570 static DEVICE_ATTR_RW(profiling);
1571 
1572 static struct attribute *panthor_attrs[] = {
1573 	&dev_attr_profiling.attr,
1574 	NULL,
1575 };
1576 
1577 ATTRIBUTE_GROUPS(panthor);
1578 
1579 static const struct of_device_id dt_match[] = {
1580 	{ .compatible = "rockchip,rk3588-mali" },
1581 	{ .compatible = "arm,mali-valhall-csf" },
1582 	{}
1583 };
1584 MODULE_DEVICE_TABLE(of, dt_match);
1585 
1586 static DEFINE_RUNTIME_DEV_PM_OPS(panthor_pm_ops,
1587 				 panthor_device_suspend,
1588 				 panthor_device_resume,
1589 				 NULL);
1590 
1591 static struct platform_driver panthor_driver = {
1592 	.probe = panthor_probe,
1593 	.remove = panthor_remove,
1594 	.driver = {
1595 		.name = "panthor",
1596 		.pm = pm_ptr(&panthor_pm_ops),
1597 		.of_match_table = dt_match,
1598 		.dev_groups = panthor_groups,
1599 	},
1600 };
1601 
1602 /*
1603  * Workqueue used to cleanup stuff.
1604  *
1605  * We create a dedicated workqueue so we can drain on unplug and
1606  * make sure all resources are freed before the module is unloaded.
1607  */
1608 struct workqueue_struct *panthor_cleanup_wq;
1609 
panthor_init(void)1610 static int __init panthor_init(void)
1611 {
1612 	int ret;
1613 
1614 	ret = panthor_mmu_pt_cache_init();
1615 	if (ret)
1616 		return ret;
1617 
1618 	panthor_cleanup_wq = alloc_workqueue("panthor-cleanup", WQ_UNBOUND, 0);
1619 	if (!panthor_cleanup_wq) {
1620 		pr_err("panthor: Failed to allocate the workqueues");
1621 		ret = -ENOMEM;
1622 		goto err_mmu_pt_cache_fini;
1623 	}
1624 
1625 	ret = platform_driver_register(&panthor_driver);
1626 	if (ret)
1627 		goto err_destroy_cleanup_wq;
1628 
1629 	return 0;
1630 
1631 err_destroy_cleanup_wq:
1632 	destroy_workqueue(panthor_cleanup_wq);
1633 
1634 err_mmu_pt_cache_fini:
1635 	panthor_mmu_pt_cache_fini();
1636 	return ret;
1637 }
1638 module_init(panthor_init);
1639 
panthor_exit(void)1640 static void __exit panthor_exit(void)
1641 {
1642 	platform_driver_unregister(&panthor_driver);
1643 	destroy_workqueue(panthor_cleanup_wq);
1644 	panthor_mmu_pt_cache_fini();
1645 }
1646 module_exit(panthor_exit);
1647 
1648 MODULE_AUTHOR("Panthor Project Developers");
1649 MODULE_DESCRIPTION("Panthor DRM Driver");
1650 MODULE_LICENSE("Dual MIT/GPL");
1651