xref: /linux/drivers/gpu/drm/tegra/submit.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020 NVIDIA Corporation */
3 
4 #include <linux/dma-fence-unwrap.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/file.h>
7 #include <linux/host1x.h>
8 #include <linux/iommu.h>
9 #include <linux/kref.h>
10 #include <linux/list.h>
11 #include <linux/nospec.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/scatterlist.h>
14 #include <linux/slab.h>
15 
16 #include <drm/drm_drv.h>
17 #include <drm/drm_file.h>
18 #include <drm/drm_syncobj.h>
19 
20 #include "drm.h"
21 #include "gem.h"
22 #include "submit.h"
23 #include "uapi.h"
24 
25 #define SUBMIT_ERR(context, fmt, ...) \
26 	dev_err_ratelimited(context->client->base.dev, \
27 		"%s: job submission failed: " fmt "\n", \
28 		current->comm, ##__VA_ARGS__)
29 
30 struct gather_bo {
31 	struct host1x_bo base;
32 
33 	struct kref ref;
34 
35 	struct device *dev;
36 	u32 *gather_data;
37 	dma_addr_t gather_data_dma;
38 	size_t gather_data_words;
39 };
40 
41 static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
42 {
43 	struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
44 
45 	kref_get(&bo->ref);
46 
47 	return host_bo;
48 }
49 
50 static void gather_bo_release(struct kref *ref)
51 {
52 	struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
53 
54 	dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma,
55 		       0);
56 	kfree(bo);
57 }
58 
59 static void gather_bo_put(struct host1x_bo *host_bo)
60 {
61 	struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
62 
63 	kref_put(&bo->ref, gather_bo_release);
64 }
65 
66 static struct host1x_bo_mapping *
67 gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)
68 {
69 	struct gather_bo *gather = container_of(bo, struct gather_bo, base);
70 	struct host1x_bo_mapping *map;
71 	int err;
72 
73 	map = kzalloc_obj(*map);
74 	if (!map)
75 		return ERR_PTR(-ENOMEM);
76 
77 	kref_init(&map->ref);
78 	map->bo = bo;
79 	map->direction = direction;
80 	map->dev = dev;
81 
82 	map->sgt = kzalloc_obj(*map->sgt);
83 	if (!map->sgt) {
84 		err = -ENOMEM;
85 		goto free;
86 	}
87 
88 	err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,
89 			      gather->gather_data_words * 4);
90 	if (err)
91 		goto free_sgt;
92 
93 	err = dma_map_sgtable(dev, map->sgt, direction, 0);
94 	if (err)
95 		goto free_sgt;
96 
97 	map->phys = sg_dma_address(map->sgt->sgl);
98 	map->size = gather->gather_data_words * 4;
99 	map->chunks = err;
100 
101 	return map;
102 
103 free_sgt:
104 	sg_free_table(map->sgt);
105 	kfree(map->sgt);
106 free:
107 	kfree(map);
108 	return ERR_PTR(err);
109 }
110 
111 static void gather_bo_unpin(struct host1x_bo_mapping *map)
112 {
113 	if (!map)
114 		return;
115 
116 	dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
117 	sg_free_table(map->sgt);
118 	kfree(map->sgt);
119 
120 	kfree(map);
121 }
122 
123 static void *gather_bo_mmap(struct host1x_bo *host_bo)
124 {
125 	struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
126 
127 	return bo->gather_data;
128 }
129 
130 static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
131 {
132 }
133 
134 static const struct host1x_bo_ops gather_bo_ops = {
135 	.get = gather_bo_get,
136 	.put = gather_bo_put,
137 	.pin = gather_bo_pin,
138 	.unpin = gather_bo_unpin,
139 	.mmap = gather_bo_mmap,
140 	.munmap = gather_bo_munmap,
141 };
142 
143 static struct tegra_drm_mapping *
144 tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)
145 {
146 	struct tegra_drm_mapping *mapping;
147 
148 	xa_lock(&context->mappings);
149 
150 	mapping = xa_load(&context->mappings, id);
151 	if (mapping)
152 		kref_get(&mapping->ref);
153 
154 	xa_unlock(&context->mappings);
155 
156 	return mapping;
157 }
158 
159 static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
160 {
161 	size_t copy_len;
162 	void *data;
163 
164 	if (check_mul_overflow(count, size, &copy_len))
165 		return ERR_PTR(-EINVAL);
166 
167 	if (copy_len > 0x4000)
168 		return ERR_PTR(-E2BIG);
169 
170 	data = vmemdup_user(from, copy_len);
171 	if (IS_ERR(data))
172 		return ERR_CAST(data);
173 
174 	return data;
175 }
176 
177 static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,
178 				   struct tegra_drm_context *context,
179 				   struct drm_tegra_channel_submit *args)
180 {
181 	struct gather_bo *bo;
182 	size_t copy_len;
183 
184 	if (args->gather_data_words == 0) {
185 		SUBMIT_ERR(context, "gather_data_words cannot be zero");
186 		return -EINVAL;
187 	}
188 
189 	if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, &copy_len)) {
190 		SUBMIT_ERR(context, "gather_data_words is too large");
191 		return -EINVAL;
192 	}
193 
194 	bo = kzalloc_obj(*bo);
195 	if (!bo) {
196 		SUBMIT_ERR(context, "failed to allocate memory for bo info");
197 		return -ENOMEM;
198 	}
199 
200 	host1x_bo_init(&bo->base, &gather_bo_ops);
201 	kref_init(&bo->ref);
202 	bo->dev = dev;
203 
204 	bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma,
205 					  GFP_KERNEL | __GFP_NOWARN, 0);
206 	if (!bo->gather_data) {
207 		SUBMIT_ERR(context, "failed to allocate memory for gather data");
208 		kfree(bo);
209 		return -ENOMEM;
210 	}
211 
212 	if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) {
213 		SUBMIT_ERR(context, "failed to copy gather data from userspace");
214 		dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0);
215 		kfree(bo);
216 		return -EFAULT;
217 	}
218 
219 	bo->gather_data_words = args->gather_data_words;
220 
221 	*pbo = bo;
222 
223 	return 0;
224 }
225 
226 static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,
227 			      struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)
228 {
229 	/* TODO check that target_offset is within bounds */
230 	dma_addr_t iova = buf->reloc.target_offset;
231 	u32 written_ptr;
232 
233 	if (mapping->bo_map)
234 		iova += mapping->iova;
235 	else
236 		iova += mapping->ctx_map->mapping->phys;
237 
238 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
239 	if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)
240 		iova |= BIT_ULL(39);
241 #endif
242 
243 	written_ptr = iova >> buf->reloc.shift;
244 
245 	if (buf->reloc.gather_offset_words >= bo->gather_data_words) {
246 		SUBMIT_ERR(context,
247 			   "relocation has too large gather offset (%u vs gather length %zu)",
248 			   buf->reloc.gather_offset_words, bo->gather_data_words);
249 		return -EINVAL;
250 	}
251 
252 	buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,
253 							    bo->gather_data_words);
254 
255 	bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
256 
257 	return 0;
258 }
259 
260 static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,
261 			       struct drm_tegra_channel_submit *args,
262 			       struct tegra_drm_submit_data *job_data)
263 {
264 	struct tegra_drm_used_mapping *mappings;
265 	struct drm_tegra_submit_buf *bufs;
266 	int err;
267 	u32 i;
268 
269 	bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs,
270 				     sizeof(*bufs));
271 	if (IS_ERR(bufs)) {
272 		SUBMIT_ERR(context, "failed to copy bufs array from userspace");
273 		return PTR_ERR(bufs);
274 	}
275 
276 	mappings = kzalloc_objs(*mappings, args->num_bufs);
277 	if (!mappings) {
278 		SUBMIT_ERR(context, "failed to allocate memory for mapping info");
279 		err = -ENOMEM;
280 		goto done;
281 	}
282 
283 	for (i = 0; i < args->num_bufs; i++) {
284 		struct drm_tegra_submit_buf *buf = &bufs[i];
285 		struct tegra_drm_mapping *mapping;
286 
287 		if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {
288 			SUBMIT_ERR(context, "invalid flag specified for buffer");
289 			err = -EINVAL;
290 			goto drop_refs;
291 		}
292 
293 		mapping = tegra_drm_mapping_get(context, buf->mapping);
294 		if (!mapping) {
295 			SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);
296 			err = -EINVAL;
297 			goto drop_refs;
298 		}
299 
300 		err = submit_write_reloc(context, bo, buf, mapping);
301 		if (err) {
302 			tegra_drm_mapping_put(mapping);
303 			goto drop_refs;
304 		}
305 
306 		mappings[i].mapping = mapping;
307 		mappings[i].flags = buf->flags;
308 	}
309 
310 	job_data->used_mappings = mappings;
311 	job_data->num_used_mappings = i;
312 
313 	err = 0;
314 
315 	goto done;
316 
317 drop_refs:
318 	while (i--)
319 		tegra_drm_mapping_put(mappings[i].mapping);
320 
321 	kfree(mappings);
322 	job_data->used_mappings = NULL;
323 
324 done:
325 	kvfree(bufs);
326 
327 	return err;
328 }
329 
330 static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,
331 			     struct xarray *syncpoints, struct drm_tegra_channel_submit *args)
332 {
333 	struct host1x_syncpt *sp;
334 
335 	if (args->syncpt.flags) {
336 		SUBMIT_ERR(context, "invalid flag specified for syncpt");
337 		return -EINVAL;
338 	}
339 
340 	/* Syncpt ref will be dropped on job release */
341 	sp = xa_load(syncpoints, args->syncpt.id);
342 	if (!sp) {
343 		SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");
344 		return -EINVAL;
345 	}
346 
347 	job->syncpt = host1x_syncpt_get(sp);
348 	job->syncpt_incrs = args->syncpt.increments;
349 
350 	return 0;
351 }
352 
353 static int submit_handle_in_fence(struct tegra_drm_context *context, struct host1x *host1x,
354 				  struct dma_fence *fence, u32 *num_waits)
355 {
356 	struct dma_fence_unwrap iter;
357 	struct host1x *fence_host1x;
358 	struct dma_fence *f;
359 	int err = 0;
360 
361 	*num_waits = 0;
362 
363 	dma_fence_unwrap_for_each(f, &iter, fence) {
364 		long wait_err;
365 
366 		if (err)
367 			continue;
368 
369 		if (dma_fence_is_signaled(f))
370 			continue;
371 
372 		err = host1x_fence_extract(f, &fence_host1x, NULL, NULL);
373 		if (!err && fence_host1x == host1x) {
374 			(*num_waits)++;
375 			continue;
376 		}
377 
378 		wait_err = dma_fence_wait_timeout(f, true, msecs_to_jiffies(10000));
379 		if (wait_err == 0) {
380 			SUBMIT_ERR(context, "wait for syncobj_in timed out");
381 			err = -ETIMEDOUT;
382 			continue;
383 		} else if (wait_err < 0) {
384 			/* In practice, -ERESTARTSYS */
385 			err = wait_err;
386 			continue;
387 		} else {
388 			err = 0;
389 		}
390 	}
391 
392 	return err;
393 }
394 
395 static void submit_job_add_prefence(struct host1x_job *job, struct dma_fence *fence, u32 class)
396 {
397 	struct dma_fence_unwrap iter;
398 	struct dma_fence *f;
399 
400 	dma_fence_unwrap_for_each(f, &iter, fence) {
401 		u32 id, threshold;
402 
403 		if (dma_fence_is_signaled(f))
404 			continue;
405 
406 		if (host1x_fence_extract(f, NULL, &id, &threshold)) {
407 			/*
408 			 * The fence may have signalled since the check above, and
409 			 * the dma_fence framework then detached its ops.
410 			 */
411 			continue;
412 		}
413 
414 		host1x_job_add_wait(job, id, threshold, false, class);
415 	}
416 }
417 
418 static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,
419 				 struct drm_tegra_submit_cmd_gather_uptr *cmd,
420 				 struct gather_bo *bo, u32 *offset,
421 				 struct tegra_drm_submit_data *job_data,
422 				 u32 *class)
423 {
424 	u32 next_offset;
425 
426 	if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {
427 		SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");
428 		return -EINVAL;
429 	}
430 
431 	/* Check for maximum gather size */
432 	if (cmd->words > 16383) {
433 		SUBMIT_ERR(context, "too many words in GATHER_UPTR command");
434 		return -EINVAL;
435 	}
436 
437 	if (check_add_overflow(*offset, cmd->words, &next_offset)) {
438 		SUBMIT_ERR(context, "too many total words in job");
439 		return -EINVAL;
440 	}
441 
442 	if (next_offset > bo->gather_data_words) {
443 		SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");
444 		return -EINVAL;
445 	}
446 
447 	if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset,
448 				  cmd->words, job_data, class)) {
449 		SUBMIT_ERR(context, "job was rejected by firewall");
450 		return -EINVAL;
451 	}
452 
453 	host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
454 
455 	*offset = next_offset;
456 
457 	return 0;
458 }
459 
460 static struct host1x_job *
461 submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,
462 		  struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,
463 		  struct xarray *syncpoints, struct dma_fence *in_fence, u32 num_prefence_waits)
464 {
465 	struct drm_tegra_submit_cmd *cmds;
466 	u32 i, gather_offset = 0, class;
467 	struct host1x_job *job;
468 	int err;
469 
470 	/* Set initial class for firewall. */
471 	class = context->client->base.class;
472 
473 	cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds,
474 				     sizeof(*cmds));
475 	if (IS_ERR(cmds)) {
476 		SUBMIT_ERR(context, "failed to copy cmds array from userspace");
477 		return ERR_CAST(cmds);
478 	}
479 
480 	job = host1x_job_alloc(context->channel, args->num_cmds + num_prefence_waits, 0, true);
481 	if (!job) {
482 		SUBMIT_ERR(context, "failed to allocate memory for job");
483 		job = ERR_PTR(-ENOMEM);
484 		goto done;
485 	}
486 
487 	err = submit_get_syncpt(context, job, syncpoints, args);
488 	if (err < 0)
489 		goto free_job;
490 
491 	job->client = &context->client->base;
492 	job->class = context->client->base.class;
493 	job->serialize = true;
494 
495 	if (in_fence)
496 		submit_job_add_prefence(job, in_fence, class);
497 
498 	for (i = 0; i < args->num_cmds; i++) {
499 		struct drm_tegra_submit_cmd *cmd = &cmds[i];
500 
501 		if (cmd->flags) {
502 			SUBMIT_ERR(context, "unknown flags given for cmd");
503 			err = -EINVAL;
504 			goto free_job;
505 		}
506 
507 		if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
508 			err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo,
509 						    &gather_offset, job_data, &class);
510 			if (err)
511 				goto free_job;
512 		} else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
513 			if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
514 				SUBMIT_ERR(context, "non-zero reserved value");
515 				err = -EINVAL;
516 				goto free_job;
517 			}
518 
519 			host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
520 					    false, class);
521 		} else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {
522 			if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
523 				SUBMIT_ERR(context, "non-zero reserved value");
524 				err = -EINVAL;
525 				goto free_job;
526 			}
527 
528 			if (cmd->wait_syncpt.id != args->syncpt.id) {
529 				SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
530 				err = -EINVAL;
531 				goto free_job;
532 			}
533 
534 			host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
535 					    true, class);
536 		} else {
537 			SUBMIT_ERR(context, "unknown cmd type");
538 			err = -EINVAL;
539 			goto free_job;
540 		}
541 	}
542 
543 	if (gather_offset == 0) {
544 		SUBMIT_ERR(context, "job must have at least one gather");
545 		err = -EINVAL;
546 		goto free_job;
547 	}
548 
549 	goto done;
550 
551 free_job:
552 	host1x_job_put(job);
553 	job = ERR_PTR(err);
554 
555 done:
556 	kvfree(cmds);
557 
558 	return job;
559 }
560 
561 static void release_job(struct host1x_job *job)
562 {
563 	struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);
564 	struct tegra_drm_submit_data *job_data = job->user_data;
565 	u32 i;
566 
567 	for (i = 0; i < job_data->num_used_mappings; i++)
568 		tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
569 
570 	if (job->memory_context) {
571 		host1x_memory_context_inactive(job->memory_context);
572 		host1x_memory_context_put(job->memory_context);
573 	}
574 
575 	kfree(job_data->used_mappings);
576 	kfree(job_data);
577 
578 	pm_runtime_mark_last_busy(client->base.dev);
579 	pm_runtime_put_autosuspend(client->base.dev);
580 }
581 
582 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
583 				   struct drm_file *file)
584 {
585 	struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
586 	struct host1x_memory_context *active_memctx = NULL;
587 	struct tegra_drm_file *fpriv = file->driver_priv;
588 	struct drm_tegra_channel_submit *args = data;
589 	struct tegra_drm_submit_data *job_data;
590 	struct drm_syncobj *syncobj = NULL;
591 	struct dma_fence *in_fence = NULL;
592 	struct tegra_drm_context *context;
593 	u32 num_prefence_waits = 0;
594 	struct host1x_job *job;
595 	struct gather_bo *bo;
596 	u32 i;
597 	int err;
598 
599 	mutex_lock(&fpriv->lock);
600 
601 	context = xa_load(&fpriv->contexts, args->context);
602 	if (!context) {
603 		mutex_unlock(&fpriv->lock);
604 		pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,
605 				   current->comm, args->context);
606 		return -EINVAL;
607 	}
608 
609 	if (context->memory_context) {
610 		err = host1x_memory_context_active(context->memory_context);
611 		if (err) {
612 			mutex_unlock(&fpriv->lock);
613 			SUBMIT_ERR(context, "failed to activate memory context");
614 			return err;
615 		}
616 
617 		active_memctx = context->memory_context;
618 	}
619 
620 	if (args->syncobj_in) {
621 		err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &in_fence);
622 		if (err) {
623 			SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);
624 			goto unlock;
625 		}
626 
627 		err = submit_handle_in_fence(context, host1x, in_fence, &num_prefence_waits);
628 		if (err)
629 			goto put_in_fence;
630 	}
631 
632 	if (args->syncobj_out) {
633 		syncobj = drm_syncobj_find(file, args->syncobj_out);
634 		if (!syncobj) {
635 			SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);
636 			err = -ENOENT;
637 			goto put_in_fence;
638 		}
639 	}
640 
641 	/* Allocate gather BO and copy gather words in. */
642 	err = submit_copy_gather_data(&bo, drm->dev, context, args);
643 	if (err)
644 		goto put_in_fence;
645 
646 	job_data = kzalloc_obj(*job_data);
647 	if (!job_data) {
648 		SUBMIT_ERR(context, "failed to allocate memory for job data");
649 		err = -ENOMEM;
650 		goto put_bo;
651 	}
652 
653 	/* Get data buffer mappings and do relocation patching. */
654 	err = submit_process_bufs(context, bo, args, job_data);
655 	if (err)
656 		goto free_job_data;
657 
658 	/* Allocate host1x_job and add gathers and waits to it. */
659 	job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints, in_fence,
660 				num_prefence_waits);
661 	if (IS_ERR(job)) {
662 		err = PTR_ERR(job);
663 		goto free_job_data;
664 	}
665 
666 	/*
667 	 * The prefence has been recorded as plain syncpoint waits, so the fence
668 	 * itself is not needed anymore.
669 	 */
670 	dma_fence_put(in_fence);
671 	in_fence = NULL;
672 
673 	/* Map gather data for Host1x. */
674 	err = host1x_job_pin(job, context->client->base.dev);
675 	if (err) {
676 		SUBMIT_ERR(context, "failed to pin job: %d", err);
677 		goto put_job;
678 	}
679 
680 	if (context->client->ops->get_streamid_offset) {
681 		err = context->client->ops->get_streamid_offset(
682 			context->client, &job->engine_streamid_offset);
683 		if (err) {
684 			SUBMIT_ERR(context, "failed to get streamid offset: %d", err);
685 			goto unpin_job;
686 		}
687 	}
688 
689 	if (context->memory_context && context->client->ops->can_use_memory_ctx) {
690 		bool supported;
691 
692 		err = context->client->ops->can_use_memory_ctx(context->client, &supported);
693 		if (err) {
694 			SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err);
695 			goto unpin_job;
696 		}
697 
698 		if (supported) {
699 			job->memory_context = active_memctx;
700 			active_memctx = NULL;
701 			host1x_memory_context_get(job->memory_context);
702 		}
703 	} else if (context->client->ops->get_streamid_offset) {
704 		/*
705 		 * Job submission will need to temporarily change stream ID,
706 		 * so need to tell it what to change it back to.
707 		 */
708 		if (!tegra_dev_iommu_get_stream_id(context->client->base.dev,
709 						   &job->engine_fallback_streamid))
710 			job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS;
711 	}
712 
713 	/* Boot engine. */
714 	err = pm_runtime_resume_and_get(context->client->base.dev);
715 	if (err < 0) {
716 		SUBMIT_ERR(context, "could not power up engine: %d", err);
717 		goto put_memory_context;
718 	}
719 
720 	job->user_data = job_data;
721 	job->release = release_job;
722 	job->timeout = 10000;
723 
724 	/*
725 	 * job_data is now part of job reference counting, so don't release
726 	 * it from here.
727 	 */
728 	job_data = NULL;
729 
730 	/* Submit job to hardware. */
731 	err = host1x_job_submit(job);
732 	if (err) {
733 		SUBMIT_ERR(context, "host1x job submission failed: %d", err);
734 		goto unpin_job;
735 	}
736 
737 	/* Return postfences to userspace and add fences to DMA reservations. */
738 	args->syncpt.value = job->syncpt_end;
739 
740 	if (syncobj) {
741 		struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true);
742 		if (IS_ERR(fence)) {
743 			err = PTR_ERR(fence);
744 			SUBMIT_ERR(context, "failed to create postfence: %d", err);
745 			goto put_job;
746 		}
747 
748 		drm_syncobj_replace_fence(syncobj, fence);
749 		dma_fence_put(fence);
750 	}
751 
752 	goto put_job;
753 
754 put_memory_context:
755 	if (job->memory_context) {
756 		host1x_memory_context_inactive(job->memory_context);
757 		host1x_memory_context_put(job->memory_context);
758 	}
759 unpin_job:
760 	host1x_job_unpin(job);
761 put_job:
762 	host1x_job_put(job);
763 free_job_data:
764 	if (job_data && job_data->used_mappings) {
765 		for (i = 0; i < job_data->num_used_mappings; i++)
766 			tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
767 
768 		kfree(job_data->used_mappings);
769 	}
770 
771 	kfree(job_data);
772 put_bo:
773 	gather_bo_put(&bo->base);
774 put_in_fence:
775 	dma_fence_put(in_fence);
776 unlock:
777 	if (syncobj)
778 		drm_syncobj_put(syncobj);
779 	if (active_memctx)
780 		host1x_memory_context_inactive(active_memctx);
781 
782 	mutex_unlock(&fpriv->lock);
783 	return err;
784 }
785