61 		  struct drm_rocket_job *job,  in rocket_copy_tasks()  argument
66 	if (job->task_struct_size < sizeof(struct drm_rocket_task))  in rocket_copy_tasks()
69 	rjob->task_count = job->task_count;  in rocket_copy_tasks()
74 	rjob->tasks = kvmalloc_objs(*rjob->tasks, job->task_count);  in rocket_copy_tasks()
84 				   u64_to_user_ptr(job->tasks) + i * job->task_struct_size,  in rocket_copy_tasks()
108 static void rocket_job_hw_submit(struct rocket_core *core, struct rocket_job *job)  in rocket_job_hw_submit()  argument
113 	/* Don't queue the job if a reset is in progress */  in rocket_job_hw_submit()
119 	task = &job->tasks[job->next_task_idx];  in rocket_job_hw_submit()
120 	job->next_task_idx++;  in rocket_job_hw_submit()
157 					struct drm_sched_job *job,  in rocket_acquire_object_fences()  argument
167 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],  in rocket_acquire_object_fences()
186 static int rocket_job_push(struct rocket_job *job)  in rocket_job_push()  argument
188 	struct rocket_device *rdev = job->rdev;  in rocket_job_push()
193 	bos = kvmalloc_array(job->in_bo_count + job->out_bo_count, sizeof(void *),  in rocket_job_push()
195 	memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *));  in rocket_job_push()
196 	memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *));  in rocket_job_push()
198 	ret = drm_gem_lock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx);  in rocket_job_push()
203 		drm_sched_job_arm(&job->base);  in rocket_job_push()
205 		job->inference_done_fence = dma_fence_get(&job->base.s_fence->finished);  in rocket_job_push()
207 		ret = rocket_acquire_object_fences(job->in_bos, job->in_bo_count, &job->base, false);  in rocket_job_push()
211 		ret = rocket_acquire_object_fences(job->out_bos, job->out_bo_count, &job->base, true);  in rocket_job_push()
215 		kref_get(&job->refcount); /* put by scheduler job completion */  in rocket_job_push()
217 		drm_sched_entity_push_job(&job->base);  in rocket_job_push()
220 	rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence);  in rocket_job_push()
223 	drm_gem_unlock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx);  in rocket_job_push()
232 	struct rocket_job *job = container_of(ref, struct rocket_job,  in rocket_job_cleanup()  local
236 	rocket_iommu_domain_put(job->domain);  in rocket_job_cleanup()
238 	dma_fence_put(job->done_fence);  in rocket_job_cleanup()
239 	dma_fence_put(job->inference_done_fence);  in rocket_job_cleanup()
241 	if (job->in_bos) {  in rocket_job_cleanup()
242 		for (i = 0; i < job->in_bo_count; i++)  in rocket_job_cleanup()
243 			drm_gem_object_put(job->in_bos[i]);  in rocket_job_cleanup()
245 		kvfree(job->in_bos);  in rocket_job_cleanup()
248 	if (job->out_bos) {  in rocket_job_cleanup()
249 		for (i = 0; i < job->out_bo_count; i++)  in rocket_job_cleanup()
250 			drm_gem_object_put(job->out_bos[i]);  in rocket_job_cleanup()
252 		kvfree(job->out_bos);  in rocket_job_cleanup()
255 	kvfree(job->tasks);  in rocket_job_cleanup()
257 	kfree(job);  in rocket_job_cleanup()
260 static void rocket_job_put(struct rocket_job *job)  in rocket_job_put()  argument
262 	kref_put(&job->refcount, rocket_job_cleanup);  in rocket_job_put()
267 	struct rocket_job *job = to_rocket_job(sched_job);  in rocket_job_free()  local
271 	rocket_job_put(job);  in rocket_job_free()
289 	struct rocket_job *job = to_rocket_job(sched_job);  in rocket_job_run()  local
290 	struct rocket_device *rdev = job->rdev;  in rocket_job_run()
295 	if (unlikely(job->base.s_fence->finished.error))  in rocket_job_run()
299 	 * Nothing to execute: can happen if the job has finished while  in rocket_job_run()
302 	if (job->next_task_idx == job->task_count)  in rocket_job_run()
309 	if (job->done_fence)  in rocket_job_run()
310 		dma_fence_put(job->done_fence);  in rocket_job_run()
311 	job->done_fence = dma_fence_get(fence);  in rocket_job_run()
317 	ret = iommu_attach_group(job->domain->domain, core->iommu_group);  in rocket_job_run()
322 		core->in_flight_job = job;  in rocket_job_run()
323 		rocket_job_hw_submit(core, job);  in rocket_job_run()
384 	struct rocket_job *job = to_rocket_job(sched_job);  in rocket_job_timedout()  local
385 	struct rocket_device *rdev = job->rdev;  in rocket_job_timedout()
388 	dev_err(core->dev, "NPU job timed out");  in rocket_job_timedout()
462 		dev_err(core->dev, "failed to request job irq");  in rocket_job_init()
535 				   struct drm_rocket_job *job)  in rocket_ioctl_submit_job()  argument
542 	if (job->task_count == 0)  in rocket_ioctl_submit_job()
559 	ret = rocket_copy_tasks(dev, file, job, rjob);  in rocket_ioctl_submit_job()
563 	ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->in_bo_handles),  in rocket_ioctl_submit_job()
564 				     job->in_bo_handle_count, &rjob->in_bos);  in rocket_ioctl_submit_job()
568 	rjob->in_bo_count = job->in_bo_handle_count;  in rocket_ioctl_submit_job()
570 	ret = drm_gem_objects_lookup(file, u64_to_user_ptr(job->out_bo_handles),  in rocket_ioctl_submit_job()
571 				     job->out_bo_handle_count, &rjob->out_bos);  in rocket_ioctl_submit_job()
575 	rjob->out_bo_count = job->out_bo_handle_count;  in rocket_ioctl_submit_job()
614 		drm_dbg(dev, "Failed to allocate incoming job array\n");  in rocket_ioctl_submit()
623 			drm_dbg(dev, "Failed to copy incoming job array\n");  in rocket_ioctl_submit()