xref: /linux/drivers/gpu/drm/msm/msm_gem_submit.c (revision a3cbcadfdfc330c28a45f06e8f92fd1d59aafa19)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include <linux/file.h>
8 #include <linux/sync_file.h>
9 #include <linux/uaccess.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_file.h>
13 #include <drm/drm_syncobj.h>
14 
15 #include "msm_drv.h"
16 #include "msm_gpu.h"
17 #include "msm_gem.h"
18 #include "msm_gpu_trace.h"
19 
20 /*
21  * Cmdstream submission:
22  */
23 
24 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
25 #define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
26 #define BO_LOCKED   0x4000
27 #define BO_PINNED   0x2000
28 
29 static struct msm_gem_submit *submit_create(struct drm_device *dev,
30 		struct msm_gpu *gpu,
31 		struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
32 		uint32_t nr_cmds)
33 {
34 	struct msm_gem_submit *submit;
35 	uint64_t sz = struct_size(submit, bos, nr_bos) +
36 				  ((u64)nr_cmds * sizeof(submit->cmd[0]));
37 
38 	if (sz > SIZE_MAX)
39 		return NULL;
40 
41 	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
42 	if (!submit)
43 		return NULL;
44 
45 	kref_init(&submit->ref);
46 	submit->dev = dev;
47 	submit->aspace = queue->ctx->aspace;
48 	submit->gpu = gpu;
49 	submit->fence = NULL;
50 	submit->cmd = (void *)&submit->bos[nr_bos];
51 	submit->queue = queue;
52 	submit->ring = gpu->rb[queue->prio];
53 	submit->fault_dumped = false;
54 
55 	/* initially, until copy_from_user() and bo lookup succeeds: */
56 	submit->nr_bos = 0;
57 	submit->nr_cmds = 0;
58 
59 	INIT_LIST_HEAD(&submit->node);
60 	INIT_LIST_HEAD(&submit->bo_list);
61 
62 	return submit;
63 }
64 
65 void __msm_gem_submit_destroy(struct kref *kref)
66 {
67 	struct msm_gem_submit *submit =
68 			container_of(kref, struct msm_gem_submit, ref);
69 	unsigned i;
70 
71 	dma_fence_put(submit->fence);
72 	put_pid(submit->pid);
73 	msm_submitqueue_put(submit->queue);
74 
75 	for (i = 0; i < submit->nr_cmds; i++)
76 		kfree(submit->cmd[i].relocs);
77 
78 	kfree(submit);
79 }
80 
81 static int submit_lookup_objects(struct msm_gem_submit *submit,
82 		struct drm_msm_gem_submit *args, struct drm_file *file)
83 {
84 	unsigned i;
85 	int ret = 0;
86 
87 	for (i = 0; i < args->nr_bos; i++) {
88 		struct drm_msm_gem_submit_bo submit_bo;
89 		void __user *userptr =
90 			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
91 
92 		/* make sure we don't have garbage flags, in case we hit
93 		 * error path before flags is initialized:
94 		 */
95 		submit->bos[i].flags = 0;
96 
97 		if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
98 			ret = -EFAULT;
99 			i = 0;
100 			goto out;
101 		}
102 
103 /* at least one of READ and/or WRITE flags should be set: */
104 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
105 
106 		if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
107 			!(submit_bo.flags & MANDATORY_FLAGS)) {
108 			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
109 			ret = -EINVAL;
110 			i = 0;
111 			goto out;
112 		}
113 
114 		submit->bos[i].handle = submit_bo.handle;
115 		submit->bos[i].flags = submit_bo.flags;
116 		/* in validate_objects() we figure out if this is true: */
117 		submit->bos[i].iova  = submit_bo.presumed;
118 	}
119 
120 	spin_lock(&file->table_lock);
121 
122 	for (i = 0; i < args->nr_bos; i++) {
123 		struct drm_gem_object *obj;
124 		struct msm_gem_object *msm_obj;
125 
126 		/* normally use drm_gem_object_lookup(), but for bulk lookup
127 		 * all under single table_lock just hit object_idr directly:
128 		 */
129 		obj = idr_find(&file->object_idr, submit->bos[i].handle);
130 		if (!obj) {
131 			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);
132 			ret = -EINVAL;
133 			goto out_unlock;
134 		}
135 
136 		msm_obj = to_msm_bo(obj);
137 
138 		if (!list_empty(&msm_obj->submit_entry)) {
139 			DRM_ERROR("handle %u at index %u already on submit list\n",
140 					submit->bos[i].handle, i);
141 			ret = -EINVAL;
142 			goto out_unlock;
143 		}
144 
145 		drm_gem_object_get(obj);
146 
147 		submit->bos[i].obj = msm_obj;
148 
149 		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
150 	}
151 
152 out_unlock:
153 	spin_unlock(&file->table_lock);
154 
155 out:
156 	submit->nr_bos = i;
157 
158 	return ret;
159 }
160 
161 static int submit_lookup_cmds(struct msm_gem_submit *submit,
162 		struct drm_msm_gem_submit *args, struct drm_file *file)
163 {
164 	unsigned i, sz;
165 	int ret = 0;
166 
167 	for (i = 0; i < args->nr_cmds; i++) {
168 		struct drm_msm_gem_submit_cmd submit_cmd;
169 		void __user *userptr =
170 			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
171 
172 		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
173 		if (ret) {
174 			ret = -EFAULT;
175 			goto out;
176 		}
177 
178 		/* validate input from userspace: */
179 		switch (submit_cmd.type) {
180 		case MSM_SUBMIT_CMD_BUF:
181 		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
182 		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
183 			break;
184 		default:
185 			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
186 			return -EINVAL;
187 		}
188 
189 		if (submit_cmd.size % 4) {
190 			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
191 					submit_cmd.size);
192 			ret = -EINVAL;
193 			goto out;
194 		}
195 
196 		submit->cmd[i].type = submit_cmd.type;
197 		submit->cmd[i].size = submit_cmd.size / 4;
198 		submit->cmd[i].offset = submit_cmd.submit_offset / 4;
199 		submit->cmd[i].idx  = submit_cmd.submit_idx;
200 		submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
201 
202 		userptr = u64_to_user_ptr(submit_cmd.relocs);
203 
204 		sz = array_size(submit_cmd.nr_relocs,
205 				sizeof(struct drm_msm_gem_submit_reloc));
206 		/* check for overflow: */
207 		if (sz == SIZE_MAX) {
208 			ret = -ENOMEM;
209 			goto out;
210 		}
211 		submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);
212 		ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
213 		if (ret) {
214 			ret = -EFAULT;
215 			goto out;
216 		}
217 	}
218 
219 out:
220 	return ret;
221 }
222 
223 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit,
224 		int i, bool backoff)
225 {
226 	struct msm_gem_object *msm_obj = submit->bos[i].obj;
227 
228 	if (submit->bos[i].flags & BO_PINNED)
229 		msm_gem_unpin_iova_locked(&msm_obj->base, submit->aspace);
230 
231 	if (submit->bos[i].flags & BO_LOCKED)
232 		dma_resv_unlock(msm_obj->base.resv);
233 
234 	if (backoff && !(submit->bos[i].flags & BO_VALID))
235 		submit->bos[i].iova = 0;
236 
237 	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
238 }
239 
240 /* This is where we make sure all the bo's are reserved and pin'd: */
241 static int submit_lock_objects(struct msm_gem_submit *submit)
242 {
243 	int contended, slow_locked = -1, i, ret = 0;
244 
245 retry:
246 	for (i = 0; i < submit->nr_bos; i++) {
247 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
248 
249 		if (slow_locked == i)
250 			slow_locked = -1;
251 
252 		contended = i;
253 
254 		if (!(submit->bos[i].flags & BO_LOCKED)) {
255 			ret = dma_resv_lock_interruptible(msm_obj->base.resv,
256 							  &submit->ticket);
257 			if (ret)
258 				goto fail;
259 			submit->bos[i].flags |= BO_LOCKED;
260 		}
261 	}
262 
263 	ww_acquire_done(&submit->ticket);
264 
265 	return 0;
266 
267 fail:
268 	for (; i >= 0; i--)
269 		submit_unlock_unpin_bo(submit, i, true);
270 
271 	if (slow_locked > 0)
272 		submit_unlock_unpin_bo(submit, slow_locked, true);
273 
274 	if (ret == -EDEADLK) {
275 		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
276 		/* we lost out in a seqno race, lock and retry.. */
277 		ret = dma_resv_lock_slow_interruptible(msm_obj->base.resv,
278 						       &submit->ticket);
279 		if (!ret) {
280 			submit->bos[contended].flags |= BO_LOCKED;
281 			slow_locked = contended;
282 			goto retry;
283 		}
284 	}
285 
286 	return ret;
287 }
288 
289 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
290 {
291 	int i, ret = 0;
292 
293 	for (i = 0; i < submit->nr_bos; i++) {
294 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
295 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
296 
297 		if (!write) {
298 			/* NOTE: _reserve_shared() must happen before
299 			 * _add_shared_fence(), which makes this a slightly
300 			 * strange place to call it.  OTOH this is a
301 			 * convenient can-fail point to hook it in.
302 			 */
303 			ret = dma_resv_reserve_shared(msm_obj->base.resv,
304 								1);
305 			if (ret)
306 				return ret;
307 		}
308 
309 		if (no_implicit)
310 			continue;
311 
312 		ret = msm_gem_sync_object(&msm_obj->base, submit->ring->fctx,
313 			write);
314 		if (ret)
315 			break;
316 	}
317 
318 	return ret;
319 }
320 
321 static int submit_pin_objects(struct msm_gem_submit *submit)
322 {
323 	int i, ret = 0;
324 
325 	submit->valid = true;
326 
327 	for (i = 0; i < submit->nr_bos; i++) {
328 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
329 		uint64_t iova;
330 
331 		/* if locking succeeded, pin bo: */
332 		ret = msm_gem_get_and_pin_iova_locked(&msm_obj->base,
333 				submit->aspace, &iova);
334 
335 		if (ret)
336 			break;
337 
338 		submit->bos[i].flags |= BO_PINNED;
339 
340 		if (iova == submit->bos[i].iova) {
341 			submit->bos[i].flags |= BO_VALID;
342 		} else {
343 			submit->bos[i].iova = iova;
344 			/* iova changed, so address in cmdstream is not valid: */
345 			submit->bos[i].flags &= ~BO_VALID;
346 			submit->valid = false;
347 		}
348 	}
349 
350 	return ret;
351 }
352 
353 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
354 		struct msm_gem_object **obj, uint64_t *iova, bool *valid)
355 {
356 	if (idx >= submit->nr_bos) {
357 		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
358 				idx, submit->nr_bos);
359 		return -EINVAL;
360 	}
361 
362 	if (obj)
363 		*obj = submit->bos[idx].obj;
364 	if (iova)
365 		*iova = submit->bos[idx].iova;
366 	if (valid)
367 		*valid = !!(submit->bos[idx].flags & BO_VALID);
368 
369 	return 0;
370 }
371 
372 /* process the reloc's and patch up the cmdstream as needed: */
373 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
374 		uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
375 {
376 	uint32_t i, last_offset = 0;
377 	uint32_t *ptr;
378 	int ret = 0;
379 
380 	if (!nr_relocs)
381 		return 0;
382 
383 	if (offset % 4) {
384 		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
385 		return -EINVAL;
386 	}
387 
388 	/* For now, just map the entire thing.  Eventually we probably
389 	 * to do it page-by-page, w/ kmap() if not vmap()d..
390 	 */
391 	ptr = msm_gem_get_vaddr_locked(&obj->base);
392 
393 	if (IS_ERR(ptr)) {
394 		ret = PTR_ERR(ptr);
395 		DBG("failed to map: %d", ret);
396 		return ret;
397 	}
398 
399 	for (i = 0; i < nr_relocs; i++) {
400 		struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
401 		uint32_t off;
402 		uint64_t iova;
403 		bool valid;
404 
405 		if (submit_reloc.submit_offset % 4) {
406 			DRM_ERROR("non-aligned reloc offset: %u\n",
407 					submit_reloc.submit_offset);
408 			ret = -EINVAL;
409 			goto out;
410 		}
411 
412 		/* offset in dwords: */
413 		off = submit_reloc.submit_offset / 4;
414 
415 		if ((off >= (obj->base.size / 4)) ||
416 				(off < last_offset)) {
417 			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
418 			ret = -EINVAL;
419 			goto out;
420 		}
421 
422 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
423 		if (ret)
424 			goto out;
425 
426 		if (valid)
427 			continue;
428 
429 		iova += submit_reloc.reloc_offset;
430 
431 		if (submit_reloc.shift < 0)
432 			iova >>= -submit_reloc.shift;
433 		else
434 			iova <<= submit_reloc.shift;
435 
436 		ptr[off] = iova | submit_reloc.or;
437 
438 		last_offset = off;
439 	}
440 
441 out:
442 	msm_gem_put_vaddr_locked(&obj->base);
443 
444 	return ret;
445 }
446 
447 static void submit_cleanup(struct msm_gem_submit *submit)
448 {
449 	unsigned i;
450 
451 	for (i = 0; i < submit->nr_bos; i++) {
452 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
453 		submit_unlock_unpin_bo(submit, i, false);
454 		list_del_init(&msm_obj->submit_entry);
455 		drm_gem_object_put_locked(&msm_obj->base);
456 	}
457 }
458 
459 
460 struct msm_submit_post_dep {
461 	struct drm_syncobj *syncobj;
462 	uint64_t point;
463 	struct dma_fence_chain *chain;
464 };
465 
466 static struct drm_syncobj **msm_wait_deps(struct drm_device *dev,
467                                           struct drm_file *file,
468                                           uint64_t in_syncobjs_addr,
469                                           uint32_t nr_in_syncobjs,
470                                           size_t syncobj_stride,
471                                           struct msm_ringbuffer *ring)
472 {
473 	struct drm_syncobj **syncobjs = NULL;
474 	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
475 	int ret = 0;
476 	uint32_t i, j;
477 
478 	syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
479 	                   GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
480 	if (!syncobjs)
481 		return ERR_PTR(-ENOMEM);
482 
483 	for (i = 0; i < nr_in_syncobjs; ++i) {
484 		uint64_t address = in_syncobjs_addr + i * syncobj_stride;
485 		struct dma_fence *fence;
486 
487 		if (copy_from_user(&syncobj_desc,
488 			           u64_to_user_ptr(address),
489 			           min(syncobj_stride, sizeof(syncobj_desc)))) {
490 			ret = -EFAULT;
491 			break;
492 		}
493 
494 		if (syncobj_desc.point &&
495 		    !drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) {
496 			ret = -EOPNOTSUPP;
497 			break;
498 		}
499 
500 		if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
501 			ret = -EINVAL;
502 			break;
503 		}
504 
505 		ret = drm_syncobj_find_fence(file, syncobj_desc.handle,
506 		                             syncobj_desc.point, 0, &fence);
507 		if (ret)
508 			break;
509 
510 		if (!dma_fence_match_context(fence, ring->fctx->context))
511 			ret = dma_fence_wait(fence, true);
512 
513 		dma_fence_put(fence);
514 		if (ret)
515 			break;
516 
517 		if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
518 			syncobjs[i] =
519 				drm_syncobj_find(file, syncobj_desc.handle);
520 			if (!syncobjs[i]) {
521 				ret = -EINVAL;
522 				break;
523 			}
524 		}
525 	}
526 
527 	if (ret) {
528 		for (j = 0; j <= i; ++j) {
529 			if (syncobjs[j])
530 				drm_syncobj_put(syncobjs[j]);
531 		}
532 		kfree(syncobjs);
533 		return ERR_PTR(ret);
534 	}
535 	return syncobjs;
536 }
537 
538 static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
539                                uint32_t nr_syncobjs)
540 {
541 	uint32_t i;
542 
543 	for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
544 		if (syncobjs[i])
545 			drm_syncobj_replace_fence(syncobjs[i], NULL);
546 	}
547 }
548 
549 static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
550                                                        struct drm_file *file,
551                                                        uint64_t syncobjs_addr,
552                                                        uint32_t nr_syncobjs,
553                                                        size_t syncobj_stride)
554 {
555 	struct msm_submit_post_dep *post_deps;
556 	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
557 	int ret = 0;
558 	uint32_t i, j;
559 
560 	post_deps = kmalloc_array(nr_syncobjs, sizeof(*post_deps),
561 	                          GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
562 	if (!post_deps)
563 		return ERR_PTR(-ENOMEM);
564 
565 	for (i = 0; i < nr_syncobjs; ++i) {
566 		uint64_t address = syncobjs_addr + i * syncobj_stride;
567 
568 		if (copy_from_user(&syncobj_desc,
569 			           u64_to_user_ptr(address),
570 			           min(syncobj_stride, sizeof(syncobj_desc)))) {
571 			ret = -EFAULT;
572 			break;
573 		}
574 
575 		post_deps[i].point = syncobj_desc.point;
576 		post_deps[i].chain = NULL;
577 
578 		if (syncobj_desc.flags) {
579 			ret = -EINVAL;
580 			break;
581 		}
582 
583 		if (syncobj_desc.point) {
584 			if (!drm_core_check_feature(dev,
585 			                            DRIVER_SYNCOBJ_TIMELINE)) {
586 				ret = -EOPNOTSUPP;
587 				break;
588 			}
589 
590 			post_deps[i].chain =
591 				kmalloc(sizeof(*post_deps[i].chain),
592 				        GFP_KERNEL);
593 			if (!post_deps[i].chain) {
594 				ret = -ENOMEM;
595 				break;
596 			}
597 		}
598 
599 		post_deps[i].syncobj =
600 			drm_syncobj_find(file, syncobj_desc.handle);
601 		if (!post_deps[i].syncobj) {
602 			ret = -EINVAL;
603 			break;
604 		}
605 	}
606 
607 	if (ret) {
608 		for (j = 0; j <= i; ++j) {
609 			kfree(post_deps[j].chain);
610 			if (post_deps[j].syncobj)
611 				drm_syncobj_put(post_deps[j].syncobj);
612 		}
613 
614 		kfree(post_deps);
615 		return ERR_PTR(ret);
616 	}
617 
618 	return post_deps;
619 }
620 
621 static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
622                                   uint32_t count, struct dma_fence *fence)
623 {
624 	uint32_t i;
625 
626 	for (i = 0; post_deps && i < count; ++i) {
627 		if (post_deps[i].chain) {
628 			drm_syncobj_add_point(post_deps[i].syncobj,
629 			                      post_deps[i].chain,
630 			                      fence, post_deps[i].point);
631 			post_deps[i].chain = NULL;
632 		} else {
633 			drm_syncobj_replace_fence(post_deps[i].syncobj,
634 			                          fence);
635 		}
636 	}
637 }
638 
639 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
640 		struct drm_file *file)
641 {
642 	static atomic_t ident = ATOMIC_INIT(0);
643 	struct msm_drm_private *priv = dev->dev_private;
644 	struct drm_msm_gem_submit *args = data;
645 	struct msm_file_private *ctx = file->driver_priv;
646 	struct msm_gem_submit *submit;
647 	struct msm_gpu *gpu = priv->gpu;
648 	struct sync_file *sync_file = NULL;
649 	struct msm_gpu_submitqueue *queue;
650 	struct msm_ringbuffer *ring;
651 	struct msm_submit_post_dep *post_deps = NULL;
652 	struct drm_syncobj **syncobjs_to_reset = NULL;
653 	int out_fence_fd = -1;
654 	struct pid *pid = get_pid(task_pid(current));
655 	bool has_ww_ticket = false;
656 	unsigned i;
657 	int ret, submitid;
658 	if (!gpu)
659 		return -ENXIO;
660 
661 	if (args->pad)
662 		return -EINVAL;
663 
664 	/* for now, we just have 3d pipe.. eventually this would need to
665 	 * be more clever to dispatch to appropriate gpu module:
666 	 */
667 	if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
668 		return -EINVAL;
669 
670 	if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
671 		return -EINVAL;
672 
673 	if (args->flags & MSM_SUBMIT_SUDO) {
674 		if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
675 		    !capable(CAP_SYS_RAWIO))
676 			return -EINVAL;
677 	}
678 
679 	queue = msm_submitqueue_get(ctx, args->queueid);
680 	if (!queue)
681 		return -ENOENT;
682 
683 	/* Get a unique identifier for the submission for logging purposes */
684 	submitid = atomic_inc_return(&ident) - 1;
685 
686 	ring = gpu->rb[queue->prio];
687 	trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid,
688 		args->nr_bos, args->nr_cmds);
689 
690 	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
691 		struct dma_fence *in_fence;
692 
693 		in_fence = sync_file_get_fence(args->fence_fd);
694 
695 		if (!in_fence)
696 			return -EINVAL;
697 
698 		/*
699 		 * Wait if the fence is from a foreign context, or if the fence
700 		 * array contains any fence from a foreign context.
701 		 */
702 		ret = 0;
703 		if (!dma_fence_match_context(in_fence, ring->fctx->context))
704 			ret = dma_fence_wait(in_fence, true);
705 
706 		dma_fence_put(in_fence);
707 		if (ret)
708 			return ret;
709 	}
710 
711 	if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
712 		syncobjs_to_reset = msm_wait_deps(dev, file,
713 		                                  args->in_syncobjs,
714 		                                  args->nr_in_syncobjs,
715 		                                  args->syncobj_stride, ring);
716 		if (IS_ERR(syncobjs_to_reset))
717 			return PTR_ERR(syncobjs_to_reset);
718 	}
719 
720 	if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
721 		post_deps = msm_parse_post_deps(dev, file,
722 		                                args->out_syncobjs,
723 		                                args->nr_out_syncobjs,
724 		                                args->syncobj_stride);
725 		if (IS_ERR(post_deps)) {
726 			ret = PTR_ERR(post_deps);
727 			goto out_post_unlock;
728 		}
729 	}
730 
731 	ret = mutex_lock_interruptible(&dev->struct_mutex);
732 	if (ret)
733 		goto out_post_unlock;
734 
735 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
736 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
737 		if (out_fence_fd < 0) {
738 			ret = out_fence_fd;
739 			goto out_unlock;
740 		}
741 	}
742 
743 	submit = submit_create(dev, gpu, queue, args->nr_bos,
744 		args->nr_cmds);
745 	if (!submit) {
746 		ret = -ENOMEM;
747 		goto out_unlock;
748 	}
749 
750 	submit->pid = pid;
751 	submit->ident = submitid;
752 
753 	if (args->flags & MSM_SUBMIT_SUDO)
754 		submit->in_rb = true;
755 
756 	ret = submit_lookup_objects(submit, args, file);
757 	if (ret)
758 		goto out_pre_pm;
759 
760 	ret = submit_lookup_cmds(submit, args, file);
761 	if (ret)
762 		goto out_pre_pm;
763 
764 	/*
765 	 * Thanks to dev_pm_opp opp_table_lock interactions with mm->mmap_sem
766 	 * in the resume path, we need to to rpm get before we lock objs.
767 	 * Which unfortunately might involve powering up the GPU sooner than
768 	 * is necessary.  But at least in the explicit fencing case, we will
769 	 * have already done all the fence waiting.
770 	 */
771 	pm_runtime_get_sync(&gpu->pdev->dev);
772 
773 	/* copy_*_user while holding a ww ticket upsets lockdep */
774 	ww_acquire_init(&submit->ticket, &reservation_ww_class);
775 	has_ww_ticket = true;
776 	ret = submit_lock_objects(submit);
777 	if (ret)
778 		goto out;
779 
780 	ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
781 	if (ret)
782 		goto out;
783 
784 	ret = submit_pin_objects(submit);
785 	if (ret)
786 		goto out;
787 
788 	for (i = 0; i < args->nr_cmds; i++) {
789 		struct msm_gem_object *msm_obj;
790 		uint64_t iova;
791 
792 		ret = submit_bo(submit, submit->cmd[i].idx,
793 				&msm_obj, &iova, NULL);
794 		if (ret)
795 			goto out;
796 
797 		if (!submit->cmd[i].size ||
798 			((submit->cmd[i].size + submit->cmd[i].offset) >
799 				msm_obj->base.size / 4)) {
800 			DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4);
801 			ret = -EINVAL;
802 			goto out;
803 		}
804 
805 		submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
806 
807 		if (submit->valid)
808 			continue;
809 
810 		ret = submit_reloc(submit, msm_obj, submit->cmd[i].offset * 4,
811 				submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
812 		if (ret)
813 			goto out;
814 	}
815 
816 	submit->nr_cmds = i;
817 
818 	submit->fence = msm_fence_alloc(ring->fctx);
819 	if (IS_ERR(submit->fence)) {
820 		ret = PTR_ERR(submit->fence);
821 		submit->fence = NULL;
822 		goto out;
823 	}
824 
825 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
826 		sync_file = sync_file_create(submit->fence);
827 		if (!sync_file) {
828 			ret = -ENOMEM;
829 			goto out;
830 		}
831 	}
832 
833 	msm_gpu_submit(gpu, submit);
834 
835 	args->fence = submit->fence->seqno;
836 
837 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
838 		fd_install(out_fence_fd, sync_file->file);
839 		args->fence_fd = out_fence_fd;
840 	}
841 
842 	msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
843 	msm_process_post_deps(post_deps, args->nr_out_syncobjs,
844 	                      submit->fence);
845 
846 
847 out:
848 	pm_runtime_put(&gpu->pdev->dev);
849 out_pre_pm:
850 	submit_cleanup(submit);
851 	if (has_ww_ticket)
852 		ww_acquire_fini(&submit->ticket);
853 	msm_gem_submit_put(submit);
854 out_unlock:
855 	if (ret && (out_fence_fd >= 0))
856 		put_unused_fd(out_fence_fd);
857 	mutex_unlock(&dev->struct_mutex);
858 
859 out_post_unlock:
860 	if (!IS_ERR_OR_NULL(post_deps)) {
861 		for (i = 0; i < args->nr_out_syncobjs; ++i) {
862 			kfree(post_deps[i].chain);
863 			drm_syncobj_put(post_deps[i].syncobj);
864 		}
865 		kfree(post_deps);
866 	}
867 
868 	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
869 		for (i = 0; i < args->nr_in_syncobjs; ++i) {
870 			if (syncobjs_to_reset[i])
871 				drm_syncobj_put(syncobjs_to_reset[i]);
872 		}
873 		kfree(syncobjs_to_reset);
874 	}
875 
876 	return ret;
877 }
878