1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7 #include <linux/file.h>
8 #include <linux/sync_file.h>
9 #include <linux/uaccess.h>
10
11 #include <drm/drm_drv.h>
12 #include <drm/drm_file.h>
13 #include <drm/drm_syncobj.h>
14
15 #include "msm_drv.h"
16 #include "msm_gpu.h"
17 #include "msm_gem.h"
18 #include "msm_gpu_trace.h"
19
20 /* For userspace errors, use DRM_UT_DRIVER.. so that userspace can enable
21 * error msgs for debugging, but we don't spam dmesg by default
22 */
23 #define SUBMIT_ERROR(err, submit, fmt, ...) \
24 UERR(err, (submit)->dev, fmt, ##__VA_ARGS__)
25
26 /*
27 * Cmdstream submission:
28 */
29
submit_create(struct drm_device * dev,struct msm_gpu * gpu,struct msm_gpu_submitqueue * queue,uint32_t nr_bos,uint32_t nr_cmds)30 static struct msm_gem_submit *submit_create(struct drm_device *dev,
31 struct msm_gpu *gpu,
32 struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
33 uint32_t nr_cmds)
34 {
35 static atomic_t ident = ATOMIC_INIT(0);
36 struct msm_gem_submit *submit;
37 uint64_t sz;
38 int ret;
39
40 sz = struct_size(submit, bos, nr_bos) +
41 ((u64)nr_cmds * sizeof(submit->cmd[0]));
42
43 if (sz > SIZE_MAX)
44 return ERR_PTR(-ENOMEM);
45
46 submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN);
47 if (!submit)
48 return ERR_PTR(-ENOMEM);
49
50 submit->hw_fence = msm_fence_alloc();
51 if (IS_ERR(submit->hw_fence)) {
52 ret = PTR_ERR(submit->hw_fence);
53 kfree(submit);
54 return ERR_PTR(ret);
55 }
56
57 ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue);
58 if (ret) {
59 kfree(submit->hw_fence);
60 kfree(submit);
61 return ERR_PTR(ret);
62 }
63
64 kref_init(&submit->ref);
65 submit->dev = dev;
66 submit->aspace = queue->ctx->aspace;
67 submit->gpu = gpu;
68 submit->cmd = (void *)&submit->bos[nr_bos];
69 submit->queue = queue;
70 submit->pid = get_pid(task_pid(current));
71 submit->ring = gpu->rb[queue->ring_nr];
72 submit->fault_dumped = false;
73
74 /* Get a unique identifier for the submission for logging purposes */
75 submit->ident = atomic_inc_return(&ident) - 1;
76
77 INIT_LIST_HEAD(&submit->node);
78
79 return submit;
80 }
81
__msm_gem_submit_destroy(struct kref * kref)82 void __msm_gem_submit_destroy(struct kref *kref)
83 {
84 struct msm_gem_submit *submit =
85 container_of(kref, struct msm_gem_submit, ref);
86 unsigned i;
87
88 /*
89 * In error paths, we could unref the submit without calling
90 * drm_sched_entity_push_job(), so msm_job_free() will never
91 * get called. Since drm_sched_job_cleanup() will NULL out
92 * s_fence, we can use that to detect this case.
93 */
94 if (submit->base.s_fence)
95 drm_sched_job_cleanup(&submit->base);
96
97 if (submit->fence_id) {
98 spin_lock(&submit->queue->idr_lock);
99 idr_remove(&submit->queue->fence_idr, submit->fence_id);
100 spin_unlock(&submit->queue->idr_lock);
101 }
102
103 dma_fence_put(submit->user_fence);
104
105 /*
106 * If the submit is freed before msm_job_run(), then hw_fence is
107 * just some pre-allocated memory, not a reference counted fence.
108 * Once the job runs and the hw_fence is initialized, it will
109 * have a refcount of at least one, since the submit holds a ref
110 * to the hw_fence.
111 */
112 if (kref_read(&submit->hw_fence->refcount) == 0) {
113 kfree(submit->hw_fence);
114 } else {
115 dma_fence_put(submit->hw_fence);
116 }
117
118 put_pid(submit->pid);
119 msm_submitqueue_put(submit->queue);
120
121 for (i = 0; i < submit->nr_cmds; i++)
122 kfree(submit->cmd[i].relocs);
123
124 kfree(submit);
125 }
126
submit_lookup_objects(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)127 static int submit_lookup_objects(struct msm_gem_submit *submit,
128 struct drm_msm_gem_submit *args, struct drm_file *file)
129 {
130 unsigned i;
131 int ret = 0;
132
133 for (i = 0; i < args->nr_bos; i++) {
134 struct drm_msm_gem_submit_bo submit_bo;
135 void __user *userptr =
136 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
137
138 /* make sure we don't have garbage flags, in case we hit
139 * error path before flags is initialized:
140 */
141 submit->bos[i].flags = 0;
142
143 if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
144 ret = -EFAULT;
145 i = 0;
146 goto out;
147 }
148
149 /* at least one of READ and/or WRITE flags should be set: */
150 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
151
152 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
153 !(submit_bo.flags & MANDATORY_FLAGS)) {
154 ret = SUBMIT_ERROR(EINVAL, submit, "invalid flags: %x\n", submit_bo.flags);
155 i = 0;
156 goto out;
157 }
158
159 submit->bos[i].handle = submit_bo.handle;
160 submit->bos[i].flags = submit_bo.flags;
161 }
162
163 spin_lock(&file->table_lock);
164
165 for (i = 0; i < args->nr_bos; i++) {
166 struct drm_gem_object *obj;
167
168 /* normally use drm_gem_object_lookup(), but for bulk lookup
169 * all under single table_lock just hit object_idr directly:
170 */
171 obj = idr_find(&file->object_idr, submit->bos[i].handle);
172 if (!obj) {
173 ret = SUBMIT_ERROR(EINVAL, submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i);
174 goto out_unlock;
175 }
176
177 drm_gem_object_get(obj);
178
179 submit->bos[i].obj = obj;
180 }
181
182 out_unlock:
183 spin_unlock(&file->table_lock);
184
185 out:
186 submit->nr_bos = i;
187
188 return ret;
189 }
190
submit_lookup_cmds(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)191 static int submit_lookup_cmds(struct msm_gem_submit *submit,
192 struct drm_msm_gem_submit *args, struct drm_file *file)
193 {
194 unsigned i;
195 size_t sz;
196 int ret = 0;
197
198 for (i = 0; i < args->nr_cmds; i++) {
199 struct drm_msm_gem_submit_cmd submit_cmd;
200 void __user *userptr =
201 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
202
203 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
204 if (ret) {
205 ret = -EFAULT;
206 goto out;
207 }
208
209 /* validate input from userspace: */
210 switch (submit_cmd.type) {
211 case MSM_SUBMIT_CMD_BUF:
212 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
213 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
214 break;
215 default:
216 return SUBMIT_ERROR(EINVAL, submit, "invalid type: %08x\n", submit_cmd.type);
217 }
218
219 if (submit_cmd.size % 4) {
220 ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer size: %u\n",
221 submit_cmd.size);
222 goto out;
223 }
224
225 submit->cmd[i].type = submit_cmd.type;
226 submit->cmd[i].size = submit_cmd.size / 4;
227 submit->cmd[i].offset = submit_cmd.submit_offset / 4;
228 submit->cmd[i].idx = submit_cmd.submit_idx;
229 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
230
231 userptr = u64_to_user_ptr(submit_cmd.relocs);
232
233 sz = array_size(submit_cmd.nr_relocs,
234 sizeof(struct drm_msm_gem_submit_reloc));
235 /* check for overflow: */
236 if (sz == SIZE_MAX) {
237 ret = -ENOMEM;
238 goto out;
239 }
240 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN);
241 if (!submit->cmd[i].relocs) {
242 ret = -ENOMEM;
243 goto out;
244 }
245 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
246 if (ret) {
247 ret = -EFAULT;
248 goto out;
249 }
250 }
251
252 out:
253 return ret;
254 }
255
256 /* This is where we make sure all the bo's are reserved and pin'd: */
submit_lock_objects(struct msm_gem_submit * submit)257 static int submit_lock_objects(struct msm_gem_submit *submit)
258 {
259 int ret;
260
261 drm_exec_init(&submit->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, submit->nr_bos);
262
263 drm_exec_until_all_locked (&submit->exec) {
264 for (unsigned i = 0; i < submit->nr_bos; i++) {
265 struct drm_gem_object *obj = submit->bos[i].obj;
266 ret = drm_exec_prepare_obj(&submit->exec, obj, 1);
267 drm_exec_retry_on_contention(&submit->exec);
268 if (ret)
269 goto error;
270 }
271 }
272
273 return 0;
274
275 error:
276 return ret;
277 }
278
submit_fence_sync(struct msm_gem_submit * submit)279 static int submit_fence_sync(struct msm_gem_submit *submit)
280 {
281 int i, ret = 0;
282
283 for (i = 0; i < submit->nr_bos; i++) {
284 struct drm_gem_object *obj = submit->bos[i].obj;
285 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
286
287 /* Otherwise userspace can ask for implicit sync to be
288 * disabled on specific buffers. This is useful for internal
289 * usermode driver managed buffers, suballocation, etc.
290 */
291 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)
292 continue;
293
294 ret = drm_sched_job_add_implicit_dependencies(&submit->base,
295 obj,
296 write);
297 if (ret)
298 break;
299 }
300
301 return ret;
302 }
303
submit_pin_objects(struct msm_gem_submit * submit)304 static int submit_pin_objects(struct msm_gem_submit *submit)
305 {
306 struct msm_drm_private *priv = submit->dev->dev_private;
307 int i, ret = 0;
308
309 for (i = 0; i < submit->nr_bos; i++) {
310 struct drm_gem_object *obj = submit->bos[i].obj;
311 struct msm_gem_vma *vma;
312
313 /* if locking succeeded, pin bo: */
314 vma = msm_gem_get_vma_locked(obj, submit->aspace);
315 if (IS_ERR(vma)) {
316 ret = PTR_ERR(vma);
317 break;
318 }
319
320 ret = msm_gem_pin_vma_locked(obj, vma);
321 if (ret)
322 break;
323
324 submit->bos[i].iova = vma->iova;
325 }
326
327 /*
328 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
329 * the LRU lock for each individual bo, while (b) avoiding holding the
330 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
331 * get_pages() which could trigger reclaim.. and if we held the LRU lock
332 * could trigger deadlock with the shrinker).
333 */
334 mutex_lock(&priv->lru.lock);
335 for (i = 0; i < submit->nr_bos; i++) {
336 msm_gem_pin_obj_locked(submit->bos[i].obj);
337 }
338 mutex_unlock(&priv->lru.lock);
339
340 submit->bos_pinned = true;
341
342 return ret;
343 }
344
submit_unpin_objects(struct msm_gem_submit * submit)345 static void submit_unpin_objects(struct msm_gem_submit *submit)
346 {
347 if (!submit->bos_pinned)
348 return;
349
350 for (int i = 0; i < submit->nr_bos; i++) {
351 struct drm_gem_object *obj = submit->bos[i].obj;
352
353 msm_gem_unpin_locked(obj);
354 }
355
356 submit->bos_pinned = false;
357 }
358
submit_attach_object_fences(struct msm_gem_submit * submit)359 static void submit_attach_object_fences(struct msm_gem_submit *submit)
360 {
361 int i;
362
363 for (i = 0; i < submit->nr_bos; i++) {
364 struct drm_gem_object *obj = submit->bos[i].obj;
365
366 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
367 dma_resv_add_fence(obj->resv, submit->user_fence,
368 DMA_RESV_USAGE_WRITE);
369 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
370 dma_resv_add_fence(obj->resv, submit->user_fence,
371 DMA_RESV_USAGE_READ);
372 }
373 }
374
submit_bo(struct msm_gem_submit * submit,uint32_t idx,struct drm_gem_object ** obj,uint64_t * iova)375 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
376 struct drm_gem_object **obj, uint64_t *iova)
377 {
378 if (idx >= submit->nr_bos) {
379 return SUBMIT_ERROR(EINVAL, submit, "invalid buffer index: %u (out of %u)\n",
380 idx, submit->nr_bos);
381 }
382
383 if (obj)
384 *obj = submit->bos[idx].obj;
385 if (iova)
386 *iova = submit->bos[idx].iova;
387
388 return 0;
389 }
390
391 /* process the reloc's and patch up the cmdstream as needed: */
submit_reloc(struct msm_gem_submit * submit,struct drm_gem_object * obj,uint32_t offset,uint32_t nr_relocs,struct drm_msm_gem_submit_reloc * relocs)392 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,
393 uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
394 {
395 uint32_t i, last_offset = 0;
396 uint32_t *ptr;
397 int ret = 0;
398
399 if (offset % 4)
400 return SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer: %u\n", offset);
401
402 /* For now, just map the entire thing. Eventually we probably
403 * to do it page-by-page, w/ kmap() if not vmap()d..
404 */
405 ptr = msm_gem_get_vaddr_locked(obj);
406
407 if (IS_ERR(ptr)) {
408 ret = PTR_ERR(ptr);
409 DBG("failed to map: %d", ret);
410 return ret;
411 }
412
413 for (i = 0; i < nr_relocs; i++) {
414 struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
415 uint32_t off;
416 uint64_t iova;
417
418 if (submit_reloc.submit_offset % 4) {
419 ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned reloc offset: %u\n",
420 submit_reloc.submit_offset);
421 goto out;
422 }
423
424 /* offset in dwords: */
425 off = submit_reloc.submit_offset / 4;
426
427 if ((off >= (obj->size / 4)) ||
428 (off < last_offset)) {
429 ret = SUBMIT_ERROR(EINVAL, submit, "invalid offset %u at reloc %u\n", off, i);
430 goto out;
431 }
432
433 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova);
434 if (ret)
435 goto out;
436
437 iova += submit_reloc.reloc_offset;
438
439 if (submit_reloc.shift < 0)
440 iova >>= -submit_reloc.shift;
441 else
442 iova <<= submit_reloc.shift;
443
444 ptr[off] = iova | submit_reloc.or;
445
446 last_offset = off;
447 }
448
449 out:
450 msm_gem_put_vaddr_locked(obj);
451
452 return ret;
453 }
454
455 /* Cleanup submit at end of ioctl. In the error case, this also drops
456 * references, unpins, and drops active refcnt. In the non-error case,
457 * this is done when the submit is retired.
458 */
submit_cleanup(struct msm_gem_submit * submit,bool error)459 static void submit_cleanup(struct msm_gem_submit *submit, bool error)
460 {
461 if (error) {
462 submit_unpin_objects(submit);
463 /* job wasn't enqueued to scheduler, so early retirement: */
464 msm_submit_retire(submit);
465 }
466
467 if (submit->exec.objects)
468 drm_exec_fini(&submit->exec);
469 }
470
msm_submit_retire(struct msm_gem_submit * submit)471 void msm_submit_retire(struct msm_gem_submit *submit)
472 {
473 int i;
474
475 for (i = 0; i < submit->nr_bos; i++) {
476 struct drm_gem_object *obj = submit->bos[i].obj;
477
478 drm_gem_object_put(obj);
479 }
480 }
481
482 struct msm_submit_post_dep {
483 struct drm_syncobj *syncobj;
484 uint64_t point;
485 struct dma_fence_chain *chain;
486 };
487
msm_parse_deps(struct msm_gem_submit * submit,struct drm_file * file,uint64_t in_syncobjs_addr,uint32_t nr_in_syncobjs,size_t syncobj_stride)488 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
489 struct drm_file *file,
490 uint64_t in_syncobjs_addr,
491 uint32_t nr_in_syncobjs,
492 size_t syncobj_stride)
493 {
494 struct drm_syncobj **syncobjs = NULL;
495 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
496 int ret = 0;
497 uint32_t i, j;
498
499 syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
500 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
501 if (!syncobjs)
502 return ERR_PTR(-ENOMEM);
503
504 for (i = 0; i < nr_in_syncobjs; ++i) {
505 uint64_t address = in_syncobjs_addr + i * syncobj_stride;
506
507 if (copy_from_user(&syncobj_desc,
508 u64_to_user_ptr(address),
509 min(syncobj_stride, sizeof(syncobj_desc)))) {
510 ret = -EFAULT;
511 break;
512 }
513
514 if (syncobj_desc.point &&
515 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {
516 ret = SUBMIT_ERROR(EOPNOTSUPP, submit, "syncobj timeline unsupported");
517 break;
518 }
519
520 if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
521 ret = SUBMIT_ERROR(EINVAL, submit, "invalid syncobj flags: %x", syncobj_desc.flags);
522 break;
523 }
524
525 ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,
526 syncobj_desc.handle, syncobj_desc.point);
527 if (ret)
528 break;
529
530 if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
531 syncobjs[i] =
532 drm_syncobj_find(file, syncobj_desc.handle);
533 if (!syncobjs[i]) {
534 ret = SUBMIT_ERROR(EINVAL, submit, "invalid syncobj handle: %u", i);
535 break;
536 }
537 }
538 }
539
540 if (ret) {
541 for (j = 0; j <= i; ++j) {
542 if (syncobjs[j])
543 drm_syncobj_put(syncobjs[j]);
544 }
545 kfree(syncobjs);
546 return ERR_PTR(ret);
547 }
548 return syncobjs;
549 }
550
msm_reset_syncobjs(struct drm_syncobj ** syncobjs,uint32_t nr_syncobjs)551 static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
552 uint32_t nr_syncobjs)
553 {
554 uint32_t i;
555
556 for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
557 if (syncobjs[i])
558 drm_syncobj_replace_fence(syncobjs[i], NULL);
559 }
560 }
561
msm_parse_post_deps(struct drm_device * dev,struct drm_file * file,uint64_t syncobjs_addr,uint32_t nr_syncobjs,size_t syncobj_stride)562 static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
563 struct drm_file *file,
564 uint64_t syncobjs_addr,
565 uint32_t nr_syncobjs,
566 size_t syncobj_stride)
567 {
568 struct msm_submit_post_dep *post_deps;
569 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
570 int ret = 0;
571 uint32_t i, j;
572
573 post_deps = kcalloc(nr_syncobjs, sizeof(*post_deps),
574 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
575 if (!post_deps)
576 return ERR_PTR(-ENOMEM);
577
578 for (i = 0; i < nr_syncobjs; ++i) {
579 uint64_t address = syncobjs_addr + i * syncobj_stride;
580
581 if (copy_from_user(&syncobj_desc,
582 u64_to_user_ptr(address),
583 min(syncobj_stride, sizeof(syncobj_desc)))) {
584 ret = -EFAULT;
585 break;
586 }
587
588 post_deps[i].point = syncobj_desc.point;
589
590 if (syncobj_desc.flags) {
591 ret = UERR(EINVAL, dev, "invalid syncobj flags");
592 break;
593 }
594
595 if (syncobj_desc.point) {
596 if (!drm_core_check_feature(dev,
597 DRIVER_SYNCOBJ_TIMELINE)) {
598 ret = UERR(EOPNOTSUPP, dev, "syncobj timeline unsupported");
599 break;
600 }
601
602 post_deps[i].chain = dma_fence_chain_alloc();
603 if (!post_deps[i].chain) {
604 ret = -ENOMEM;
605 break;
606 }
607 }
608
609 post_deps[i].syncobj =
610 drm_syncobj_find(file, syncobj_desc.handle);
611 if (!post_deps[i].syncobj) {
612 ret = UERR(EINVAL, dev, "invalid syncobj handle");
613 break;
614 }
615 }
616
617 if (ret) {
618 for (j = 0; j <= i; ++j) {
619 dma_fence_chain_free(post_deps[j].chain);
620 if (post_deps[j].syncobj)
621 drm_syncobj_put(post_deps[j].syncobj);
622 }
623
624 kfree(post_deps);
625 return ERR_PTR(ret);
626 }
627
628 return post_deps;
629 }
630
msm_process_post_deps(struct msm_submit_post_dep * post_deps,uint32_t count,struct dma_fence * fence)631 static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
632 uint32_t count, struct dma_fence *fence)
633 {
634 uint32_t i;
635
636 for (i = 0; post_deps && i < count; ++i) {
637 if (post_deps[i].chain) {
638 drm_syncobj_add_point(post_deps[i].syncobj,
639 post_deps[i].chain,
640 fence, post_deps[i].point);
641 post_deps[i].chain = NULL;
642 } else {
643 drm_syncobj_replace_fence(post_deps[i].syncobj,
644 fence);
645 }
646 }
647 }
648
msm_ioctl_gem_submit(struct drm_device * dev,void * data,struct drm_file * file)649 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
650 struct drm_file *file)
651 {
652 struct msm_drm_private *priv = dev->dev_private;
653 struct drm_msm_gem_submit *args = data;
654 struct msm_file_private *ctx = file->driver_priv;
655 struct msm_gem_submit *submit = NULL;
656 struct msm_gpu *gpu = priv->gpu;
657 struct msm_gpu_submitqueue *queue;
658 struct msm_ringbuffer *ring;
659 struct msm_submit_post_dep *post_deps = NULL;
660 struct drm_syncobj **syncobjs_to_reset = NULL;
661 struct sync_file *sync_file = NULL;
662 int out_fence_fd = -1;
663 unsigned i;
664 int ret;
665
666 if (!gpu)
667 return -ENXIO;
668
669 if (args->pad)
670 return -EINVAL;
671
672 if (unlikely(!ctx->aspace) && !capable(CAP_SYS_RAWIO)) {
673 DRM_ERROR_RATELIMITED("IOMMU support or CAP_SYS_RAWIO required!\n");
674 return -EPERM;
675 }
676
677 /* for now, we just have 3d pipe.. eventually this would need to
678 * be more clever to dispatch to appropriate gpu module:
679 */
680 if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
681 return UERR(EINVAL, dev, "invalid pipe");
682
683 if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
684 return UERR(EINVAL, dev, "invalid flags");
685
686 if (args->flags & MSM_SUBMIT_SUDO) {
687 if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
688 !capable(CAP_SYS_RAWIO))
689 return -EINVAL;
690 }
691
692 queue = msm_submitqueue_get(ctx, args->queueid);
693 if (!queue)
694 return -ENOENT;
695
696 ring = gpu->rb[queue->ring_nr];
697
698 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
699 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
700 if (out_fence_fd < 0) {
701 ret = out_fence_fd;
702 goto out_post_unlock;
703 }
704 }
705
706 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
707 if (IS_ERR(submit)) {
708 ret = PTR_ERR(submit);
709 goto out_post_unlock;
710 }
711
712 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
713 args->nr_bos, args->nr_cmds);
714
715 ret = mutex_lock_interruptible(&queue->lock);
716 if (ret)
717 goto out_post_unlock;
718
719 if (args->flags & MSM_SUBMIT_SUDO)
720 submit->in_rb = true;
721
722 if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
723 struct dma_fence *in_fence;
724
725 in_fence = sync_file_get_fence(args->fence_fd);
726
727 if (!in_fence) {
728 ret = UERR(EINVAL, dev, "invalid in-fence");
729 goto out_unlock;
730 }
731
732 ret = drm_sched_job_add_dependency(&submit->base, in_fence);
733 if (ret)
734 goto out_unlock;
735 }
736
737 if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
738 syncobjs_to_reset = msm_parse_deps(submit, file,
739 args->in_syncobjs,
740 args->nr_in_syncobjs,
741 args->syncobj_stride);
742 if (IS_ERR(syncobjs_to_reset)) {
743 ret = PTR_ERR(syncobjs_to_reset);
744 goto out_unlock;
745 }
746 }
747
748 if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
749 post_deps = msm_parse_post_deps(dev, file,
750 args->out_syncobjs,
751 args->nr_out_syncobjs,
752 args->syncobj_stride);
753 if (IS_ERR(post_deps)) {
754 ret = PTR_ERR(post_deps);
755 goto out_unlock;
756 }
757 }
758
759 ret = submit_lookup_objects(submit, args, file);
760 if (ret)
761 goto out;
762
763 ret = submit_lookup_cmds(submit, args, file);
764 if (ret)
765 goto out;
766
767 /* copy_*_user while holding a ww ticket upsets lockdep */
768 ret = submit_lock_objects(submit);
769 if (ret)
770 goto out;
771
772 if (!(args->flags & MSM_SUBMIT_NO_IMPLICIT)) {
773 ret = submit_fence_sync(submit);
774 if (ret)
775 goto out;
776 }
777
778 ret = submit_pin_objects(submit);
779 if (ret)
780 goto out;
781
782 for (i = 0; i < args->nr_cmds; i++) {
783 struct drm_gem_object *obj;
784 uint64_t iova;
785
786 ret = submit_bo(submit, submit->cmd[i].idx, &obj, &iova);
787 if (ret)
788 goto out;
789
790 if (!submit->cmd[i].size ||
791 (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) {
792 ret = UERR(EINVAL, dev, "invalid cmdstream size: %u\n",
793 submit->cmd[i].size * 4);
794 goto out;
795 }
796
797 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
798
799 if (likely(!submit->cmd[i].nr_relocs))
800 continue;
801
802 if (!gpu->allow_relocs) {
803 ret = UERR(EINVAL, dev, "relocs not allowed\n");
804 goto out;
805 }
806
807 ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,
808 submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
809 if (ret)
810 goto out;
811 }
812
813 submit->nr_cmds = i;
814
815 idr_preload(GFP_KERNEL);
816
817 spin_lock(&queue->idr_lock);
818
819 /*
820 * If using userspace provided seqno fence, validate that the id
821 * is available before arming sched job. Since access to fence_idr
822 * is serialized on the queue lock, the slot should be still avail
823 * after the job is armed
824 */
825 if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
826 (!args->fence || idr_find(&queue->fence_idr, args->fence))) {
827 spin_unlock(&queue->idr_lock);
828 idr_preload_end();
829 ret = UERR(EINVAL, dev, "invalid in-fence-sn");
830 goto out;
831 }
832
833 drm_sched_job_arm(&submit->base);
834
835 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
836
837 if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
838 /*
839 * Userspace has assigned the seqno fence that it wants
840 * us to use. It is an error to pick a fence sequence
841 * number that is not available.
842 */
843 submit->fence_id = args->fence;
844 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
845 &submit->fence_id, submit->fence_id,
846 GFP_NOWAIT);
847 /*
848 * We've already validated that the fence_id slot is valid,
849 * so if idr_alloc_u32 failed, it is a kernel bug
850 */
851 WARN_ON(ret);
852 } else {
853 /*
854 * Allocate an id which can be used by WAIT_FENCE ioctl to map
855 * back to the underlying fence.
856 */
857 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
858 submit->user_fence, 1,
859 INT_MAX, GFP_NOWAIT);
860 }
861
862 spin_unlock(&queue->idr_lock);
863 idr_preload_end();
864
865 if (submit->fence_id < 0) {
866 ret = submit->fence_id;
867 submit->fence_id = 0;
868 }
869
870 if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
871 sync_file = sync_file_create(submit->user_fence);
872 if (!sync_file) {
873 ret = -ENOMEM;
874 } else {
875 fd_install(out_fence_fd, sync_file->file);
876 args->fence_fd = out_fence_fd;
877 }
878 }
879
880 if (ret)
881 goto out;
882
883 submit_attach_object_fences(submit);
884
885 /* The scheduler owns a ref now: */
886 msm_gem_submit_get(submit);
887
888 msm_rd_dump_submit(priv->rd, submit, NULL);
889
890 drm_sched_entity_push_job(&submit->base);
891
892 args->fence = submit->fence_id;
893 queue->last_fence = submit->fence_id;
894
895 msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
896 msm_process_post_deps(post_deps, args->nr_out_syncobjs,
897 submit->user_fence);
898
899
900 out:
901 submit_cleanup(submit, !!ret);
902 out_unlock:
903 mutex_unlock(&queue->lock);
904 out_post_unlock:
905 if (ret && (out_fence_fd >= 0)) {
906 put_unused_fd(out_fence_fd);
907 if (sync_file)
908 fput(sync_file->file);
909 }
910
911 if (!IS_ERR_OR_NULL(submit)) {
912 msm_gem_submit_put(submit);
913 } else {
914 /*
915 * If the submit hasn't yet taken ownership of the queue
916 * then we need to drop the reference ourself:
917 */
918 msm_submitqueue_put(queue);
919 }
920 if (!IS_ERR_OR_NULL(post_deps)) {
921 for (i = 0; i < args->nr_out_syncobjs; ++i) {
922 kfree(post_deps[i].chain);
923 drm_syncobj_put(post_deps[i].syncobj);
924 }
925 kfree(post_deps);
926 }
927
928 if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
929 for (i = 0; i < args->nr_in_syncobjs; ++i) {
930 if (syncobjs_to_reset[i])
931 drm_syncobj_put(syncobjs_to_reset[i]);
932 }
933 kfree(syncobjs_to_reset);
934 }
935
936 return ret;
937 }
938