25 #define SUBMIT_ERROR(err, submit, fmt, ...) \  argument
26 	UERR(err, (submit)->dev, fmt, ##__VA_ARGS__)
38 	struct msm_gem_submit *submit;  in submit_create()  local
42 	sz = struct_size(submit, bos, nr_bos) +  in submit_create()
43 			((u64)nr_cmds * sizeof(submit->cmd[0]));  in submit_create()
48 	submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN);  in submit_create()
49 	if (!submit)  in submit_create()
52 	submit->hw_fence = msm_fence_alloc();  in submit_create()
53 	if (IS_ERR(submit->hw_fence)) {  in submit_create()
54 		ret = PTR_ERR(submit->hw_fence);  in submit_create()
55 		kfree(submit);  in submit_create()
59 	ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue,  in submit_create()
62 		kfree(submit->hw_fence);  in submit_create()
63 		kfree(submit);  in submit_create()
67 	kref_init(&submit->ref);  in submit_create()
68 	submit->dev = dev;  in submit_create()
69 	submit->vm = msm_context_vm(dev, queue->ctx);  in submit_create()
70 	submit->gpu = gpu;  in submit_create()
71 	submit->cmd = (void *)&submit->bos[nr_bos];  in submit_create()
72 	submit->queue = queue;  in submit_create()
73 	submit->pid = get_pid(task_pid(current));  in submit_create()
74 	submit->ring = gpu->rb[queue->ring_nr];  in submit_create()
75 	submit->fault_dumped = false;  in submit_create()
78 	submit->ident = atomic_inc_return(&ident) - 1;  in submit_create()
80 	INIT_LIST_HEAD(&submit->node);  in submit_create()
82 	return submit;  in submit_create()
87 	struct msm_gem_submit *submit =  in __msm_gem_submit_destroy()  local
92 	 * In error paths, we could unref the submit without calling  in __msm_gem_submit_destroy()
97 	if (submit->base.s_fence)  in __msm_gem_submit_destroy()
98 		drm_sched_job_cleanup(&submit->base);  in __msm_gem_submit_destroy()
100 	if (submit->fence_id) {  in __msm_gem_submit_destroy()
101 		spin_lock(&submit->queue->idr_lock);  in __msm_gem_submit_destroy()
102 		idr_remove(&submit->queue->fence_idr, submit->fence_id);  in __msm_gem_submit_destroy()
103 		spin_unlock(&submit->queue->idr_lock);  in __msm_gem_submit_destroy()
106 	dma_fence_put(submit->user_fence);  in __msm_gem_submit_destroy()
109 	 * If the submit is freed before msm_job_run(), then hw_fence is  in __msm_gem_submit_destroy()
112 	 * have a refcount of at least one, since the submit holds a ref  in __msm_gem_submit_destroy()
115 	if (kref_read(&submit->hw_fence->refcount) == 0) {  in __msm_gem_submit_destroy()
116 		kfree(submit->hw_fence);  in __msm_gem_submit_destroy()
118 		dma_fence_put(submit->hw_fence);  in __msm_gem_submit_destroy()
121 	put_pid(submit->pid);  in __msm_gem_submit_destroy()
122 	msm_submitqueue_put(submit->queue);  in __msm_gem_submit_destroy()
124 	for (i = 0; i < submit->nr_cmds; i++)  in __msm_gem_submit_destroy()
125 		kfree(submit->cmd[i].relocs);  in __msm_gem_submit_destroy()
127 	kfree(submit);  in __msm_gem_submit_destroy()
130 static int submit_lookup_objects(struct msm_gem_submit *submit,  in submit_lookup_objects()  argument
144 		submit->bos[i].flags = 0;  in submit_lookup_objects()
157 			ret = SUBMIT_ERROR(EINVAL, submit, "invalid flags: %x\n", submit_bo.flags);  in submit_lookup_objects()
162 		submit->bos[i].handle = submit_bo.handle;  in submit_lookup_objects()
163 		submit->bos[i].flags = submit_bo.flags;  in submit_lookup_objects()
174 		obj = idr_find(&file->object_idr, submit->bos[i].handle);  in submit_lookup_objects()
176 			ret = SUBMIT_ERROR(EINVAL, submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i);  in submit_lookup_objects()
182 		submit->bos[i].obj = obj;  in submit_lookup_objects()
189 	submit->nr_bos = i;  in submit_lookup_objects()
194 static int submit_lookup_cmds(struct msm_gem_submit *submit,  in submit_lookup_cmds()  argument
220 			return SUBMIT_ERROR(EINVAL, submit, "invalid type: %08x\n", submit_cmd.type);  in submit_lookup_cmds()
224 			ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer size: %u\n",  in submit_lookup_cmds()
231 				ret = SUBMIT_ERROR(EINVAL, submit, "nr_relocs must be zero");  in submit_lookup_cmds()
236 				ret = SUBMIT_ERROR(EINVAL, submit, "submit_idx/offset must be zero");  in submit_lookup_cmds()
240 			submit->cmd[i].iova = submit_cmd.iova;  in submit_lookup_cmds()
243 		submit->cmd[i].type = submit_cmd.type;  in submit_lookup_cmds()
244 		submit->cmd[i].size = submit_cmd.size / 4;  in submit_lookup_cmds()
245 		submit->cmd[i].offset = submit_cmd.submit_offset / 4;  in submit_lookup_cmds()
246 		submit->cmd[i].idx  = submit_cmd.submit_idx;  in submit_lookup_cmds()
247 		submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;  in submit_lookup_cmds()
258 		submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN);  in submit_lookup_cmds()
259 		if (!submit->cmd[i].relocs) {  in submit_lookup_cmds()
263 		ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);  in submit_lookup_cmds()
274 static int submit_lock_objects_vmbind(struct msm_gem_submit *submit)  in submit_lock_objects_vmbind()  argument
277 	struct drm_exec *exec = &submit->exec;  in submit_lock_objects_vmbind()
280 	drm_exec_init(&submit->exec, flags, submit->nr_bos);  in submit_lock_objects_vmbind()
282 	drm_exec_until_all_locked (&submit->exec) {  in submit_lock_objects_vmbind()
283 		ret = drm_gpuvm_prepare_vm(submit->vm, exec, 1);  in submit_lock_objects_vmbind()
288 		ret = drm_gpuvm_prepare_objects(submit->vm, exec, 1);  in submit_lock_objects_vmbind()
298 static int submit_lock_objects(struct msm_gem_submit *submit)  in submit_lock_objects()  argument
303 	if (msm_context_is_vmbind(submit->queue->ctx))  in submit_lock_objects()
304 		return submit_lock_objects_vmbind(submit);  in submit_lock_objects()
306 	drm_exec_init(&submit->exec, flags, submit->nr_bos);  in submit_lock_objects()
308 	drm_exec_until_all_locked (&submit->exec) {  in submit_lock_objects()
309 		ret = drm_exec_lock_obj(&submit->exec,  in submit_lock_objects()
310 					drm_gpuvm_resv_obj(submit->vm));  in submit_lock_objects()
311 		drm_exec_retry_on_contention(&submit->exec);  in submit_lock_objects()
314 		for (unsigned i = 0; i < submit->nr_bos; i++) {  in submit_lock_objects()
315 			struct drm_gem_object *obj = submit->bos[i].obj;  in submit_lock_objects()
316 			ret = drm_exec_prepare_obj(&submit->exec, obj, 1);  in submit_lock_objects()
317 			drm_exec_retry_on_contention(&submit->exec);  in submit_lock_objects()
326 static int submit_fence_sync(struct msm_gem_submit *submit)  in submit_fence_sync()  argument
330 	for (i = 0; i < submit->nr_bos; i++) {  in submit_fence_sync()
331 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_fence_sync()
332 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;  in submit_fence_sync()
338 		if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)  in submit_fence_sync()
341 		ret = drm_sched_job_add_implicit_dependencies(&submit->base,  in submit_fence_sync()
351 static int submit_pin_objects(struct msm_gem_submit *submit)  in submit_pin_objects()  argument
353 	struct msm_drm_private *priv = submit->dev->dev_private;  in submit_pin_objects()
356 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
357 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_pin_objects()
361 		vma = msm_gem_get_vma_locked(obj, submit->vm);  in submit_pin_objects()
371 		submit->bos[i].vm_bo = drm_gpuvm_bo_get(vma->vm_bo);  in submit_pin_objects()
372 		submit->bos[i].iova = vma->va.addr;  in submit_pin_objects()
383 	for (i = 0; i < submit->nr_bos; i++) {  in submit_pin_objects()
384 		msm_gem_pin_obj_locked(submit->bos[i].obj);  in submit_pin_objects()
388 	submit->bos_pinned = true;  in submit_pin_objects()
393 static void submit_unpin_objects(struct msm_gem_submit *submit)  in submit_unpin_objects()  argument
395 	if (!submit->bos_pinned)  in submit_unpin_objects()
398 	for (int i = 0; i < submit->nr_bos; i++) {  in submit_unpin_objects()
399 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_unpin_objects()
404 	submit->bos_pinned = false;  in submit_unpin_objects()
407 static void submit_attach_object_fences(struct msm_gem_submit *submit)  in submit_attach_object_fences()  argument
409 	struct msm_gem_vm *vm = to_msm_vm(submit->vm);  in submit_attach_object_fences()
412 	if (msm_context_is_vmbind(submit->queue->ctx)) {  in submit_attach_object_fences()
413 		drm_gpuvm_resv_add_fence(submit->vm, &submit->exec,  in submit_attach_object_fences()
414 					 submit->user_fence,  in submit_attach_object_fences()
419 		vm->last_fence = dma_fence_unwrap_merge(submit->user_fence, last_fence);  in submit_attach_object_fences()
425 	for (unsigned i = 0; i < submit->nr_bos; i++) {  in submit_attach_object_fences()
426 		struct drm_gem_object *obj = submit->bos[i].obj;  in submit_attach_object_fences()
428 		if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)  in submit_attach_object_fences()
429 			dma_resv_add_fence(obj->resv, submit->user_fence,  in submit_attach_object_fences()
431 		else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)  in submit_attach_object_fences()
432 			dma_resv_add_fence(obj->resv, submit->user_fence,  in submit_attach_object_fences()
437 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,  in submit_bo()  argument
440 	if (idx >= submit->nr_bos) {  in submit_bo()
441 		return SUBMIT_ERROR(EINVAL, submit, "invalid buffer index: %u (out of %u)\n",  in submit_bo()
442 				    idx, submit->nr_bos);  in submit_bo()
446 		*obj = submit->bos[idx].obj;  in submit_bo()
448 		*iova = submit->bos[idx].iova;  in submit_bo()
454 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,  in submit_reloc()  argument
462 		return SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer: %u\n", offset);  in submit_reloc()
481 			ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned reloc offset: %u\n",  in submit_reloc()
491 			ret = SUBMIT_ERROR(EINVAL, submit, "invalid offset %u at reloc %u\n", off, i);  in submit_reloc()
495 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova);  in submit_reloc()
517 /* Cleanup submit at end of ioctl.  In the error case, this also drops
519  * this is done when the submit is retired.
521 static void submit_cleanup(struct msm_gem_submit *submit, bool error)  in submit_cleanup()  argument
524 		submit_unpin_objects(submit);  in submit_cleanup()
526 	if (submit->exec.objects)  in submit_cleanup()
527 		drm_exec_fini(&submit->exec);  in submit_cleanup()
531 		msm_submit_retire(submit);  in submit_cleanup()
534 void msm_submit_retire(struct msm_gem_submit *submit)  in msm_submit_retire()  argument
538 	for (i = 0; i < submit->nr_bos; i++) {  in msm_submit_retire()
539 		struct drm_gem_object *obj = submit->bos[i].obj;  in msm_submit_retire()
540 		struct drm_gpuvm_bo *vm_bo = submit->bos[i].vm_bo;  in msm_submit_retire()
555 	struct msm_gem_submit *submit = NULL;  in msm_ioctl_gem_submit()  local
610 	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds,  in msm_ioctl_gem_submit()
612 	if (IS_ERR(submit)) {  in msm_ioctl_gem_submit()
613 		ret = PTR_ERR(submit);  in msm_ioctl_gem_submit()
617 	trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,  in msm_ioctl_gem_submit()
625 		submit->in_rb = true;  in msm_ioctl_gem_submit()
637 		ret = drm_sched_job_add_dependency(&submit->base, in_fence);  in msm_ioctl_gem_submit()
643 		syncobjs_to_reset = msm_syncobj_parse_deps(dev, &submit->base,  in msm_ioctl_gem_submit()
664 	ret = submit_lookup_objects(submit, args, file);  in msm_ioctl_gem_submit()
668 	ret = submit_lookup_cmds(submit, args, file);  in msm_ioctl_gem_submit()
673 	ret = submit_lock_objects(submit);  in msm_ioctl_gem_submit()
678 		ret = submit_fence_sync(submit);  in msm_ioctl_gem_submit()
683 	ret = submit_pin_objects(submit);  in msm_ioctl_gem_submit()
693 		ret = submit_bo(submit, submit->cmd[i].idx, &obj, &iova);  in msm_ioctl_gem_submit()
697 		if (!submit->cmd[i].size ||  in msm_ioctl_gem_submit()
698 		    (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) {  in msm_ioctl_gem_submit()
700 				   submit->cmd[i].size * 4);  in msm_ioctl_gem_submit()
704 		submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);  in msm_ioctl_gem_submit()
706 		if (likely(!submit->cmd[i].nr_relocs))  in msm_ioctl_gem_submit()
714 		ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,  in msm_ioctl_gem_submit()
715 				submit->cmd[i].nr_relocs, submit->cmd[i].relocs);  in msm_ioctl_gem_submit()
720 	submit->nr_cmds = args->nr_cmds;  in msm_ioctl_gem_submit()
740 	drm_sched_job_arm(&submit->base);  in msm_ioctl_gem_submit()
742 	submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);  in msm_ioctl_gem_submit()
750 		submit->fence_id = args->fence;  in msm_ioctl_gem_submit()
751 		ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,  in msm_ioctl_gem_submit()
752 				    &submit->fence_id, submit->fence_id,  in msm_ioctl_gem_submit()
764 		submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,  in msm_ioctl_gem_submit()
765 						    submit->user_fence, 1,  in msm_ioctl_gem_submit()
772 	if (submit->fence_id < 0) {  in msm_ioctl_gem_submit()
773 		ret = submit->fence_id;  in msm_ioctl_gem_submit()
774 		submit->fence_id = 0;  in msm_ioctl_gem_submit()
778 		sync_file = sync_file_create(submit->user_fence);  in msm_ioctl_gem_submit()
786 	submit_attach_object_fences(submit);  in msm_ioctl_gem_submit()
791 		 * just the BOs attached to the submit.  In that case we don't  in msm_ioctl_gem_submit()
793 		 * what BOs are associated with the submit.  in msm_ioctl_gem_submit()
795 		ret = drm_gpuvm_validate(submit->vm, &submit->exec);  in msm_ioctl_gem_submit()
801 	msm_gem_submit_get(submit);  in msm_ioctl_gem_submit()
803 	msm_rd_dump_submit(priv->rd, submit, NULL);  in msm_ioctl_gem_submit()
805 	drm_sched_entity_push_job(&submit->base);  in msm_ioctl_gem_submit()
807 	args->fence = submit->fence_id;  in msm_ioctl_gem_submit()
808 	queue->last_fence = submit->fence_id;  in msm_ioctl_gem_submit()
811 	msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, submit->user_fence);  in msm_ioctl_gem_submit()
814 	submit_cleanup(submit, !!ret);  in msm_ioctl_gem_submit()
828 	if (!IS_ERR_OR_NULL(submit)) {  in msm_ioctl_gem_submit()
829 		msm_gem_submit_put(submit);  in msm_ioctl_gem_submit()
832 		 * If the submit hasn't yet taken ownership of the queue  in msm_ioctl_gem_submit()