29 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_ring_db()  argument
31 	ivpu_hw_db_set(vdev, cmdq->db_id);  in ivpu_cmdq_ring_db()
35 					  struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_preemption_buffers_create()  argument
44 	cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user,  in ivpu_preemption_buffers_create()
46 	if (!cmdq->primary_preempt_buf) {  in ivpu_preemption_buffers_create()
51 	cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma,  in ivpu_preemption_buffers_create()
53 	if (!cmdq->secondary_preempt_buf) {  in ivpu_preemption_buffers_create()
61 	ivpu_bo_free(cmdq->primary_preempt_buf);  in ivpu_preemption_buffers_create()
62 	cmdq->primary_preempt_buf = NULL;  in ivpu_preemption_buffers_create()
67 					 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_preemption_buffers_free()  argument
72 	if (cmdq->primary_preempt_buf)  in ivpu_preemption_buffers_free()
73 		ivpu_bo_free(cmdq->primary_preempt_buf);  in ivpu_preemption_buffers_free()
74 	if (cmdq->secondary_preempt_buf)  in ivpu_preemption_buffers_free()
75 		ivpu_bo_free(cmdq->secondary_preempt_buf);  in ivpu_preemption_buffers_free()
81 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_alloc()  local
84 	cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);  in ivpu_cmdq_alloc()
85 	if (!cmdq)  in ivpu_cmdq_alloc()
88 	cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);  in ivpu_cmdq_alloc()
89 	if (!cmdq->mem)  in ivpu_cmdq_alloc()
92 	ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);  in ivpu_cmdq_alloc()
96 	return cmdq;  in ivpu_cmdq_alloc()
99 	kfree(cmdq);  in ivpu_cmdq_alloc()
105  * @cmdq: Pointer to the command queue structure.
109 static inline u32 ivpu_cmdq_get_entry_count(struct ivpu_cmdq *cmdq)  in ivpu_cmdq_get_entry_count()  argument
111 	size_t size = ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header);  in ivpu_cmdq_get_entry_count()
140 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_free()  argument
142 	ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);  in ivpu_cmdq_free()
143 	ivpu_bo_free(cmdq->mem);  in ivpu_cmdq_free()
144 	kfree(cmdq);  in ivpu_cmdq_free()
150 	struct ivpu_cmdq *cmdq = NULL;  in ivpu_cmdq_create()  local
155 	cmdq = ivpu_cmdq_alloc(file_priv);  in ivpu_cmdq_create()
156 	if (!cmdq) {  in ivpu_cmdq_create()
160 	ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit,  in ivpu_cmdq_create()
167 	cmdq->entry_count = ivpu_cmdq_get_entry_count(cmdq);  in ivpu_cmdq_create()
168 	cmdq->priority = priority;  in ivpu_cmdq_create()
170 	cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);  in ivpu_cmdq_create()
171 	cmdq->jobq->header.engine_idx = VPU_ENGINE_COMPUTE;  in ivpu_cmdq_create()
172 	cmdq->jobq->header.flags = ivpu_cmdq_get_flags(vdev, flags);  in ivpu_cmdq_create()
175 		 cmdq->id, file_priv->ctx.id, cmdq->jobq->header.flags);  in ivpu_cmdq_create()
176 	return cmdq;  in ivpu_cmdq_create()
179 	ivpu_cmdq_free(file_priv, cmdq);  in ivpu_cmdq_create()
183 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,  in ivpu_hws_cmdq_init()  argument
189 	ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id,  in ivpu_hws_cmdq_init()
191 				       cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_hws_cmdq_init()
195 	ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id,  in ivpu_hws_cmdq_init()
203 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_register_db()  argument
208 	ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next,  in ivpu_register_db()
216 		ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id,  in ivpu_register_db()
217 					       cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_register_db()
219 		ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,  in ivpu_register_db()
220 					   cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_register_db()
223 		ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n",  in ivpu_register_db()
224 			 cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority);  in ivpu_register_db()
226 		xa_erase(&vdev->db_xa, cmdq->db_id);  in ivpu_register_db()
239 static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_register()  argument
246 	if (cmdq->db_id)  in ivpu_cmdq_register()
249 	ivpu_cmdq_jobq_reset(vdev, cmdq->jobq);  in ivpu_cmdq_register()
252 		ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, cmdq->priority);  in ivpu_cmdq_register()
257 	ret = ivpu_register_db(file_priv, cmdq);  in ivpu_cmdq_register()
264 static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_unregister()  argument
271 	if (!cmdq->db_id)  in ivpu_cmdq_unregister()
274 	ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);  in ivpu_cmdq_unregister()
276 		ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);  in ivpu_cmdq_unregister()
279 		ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id);  in ivpu_cmdq_unregister()
282 				 cmdq->id, file_priv->ctx.id);  in ivpu_cmdq_unregister()
285 	xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);  in ivpu_cmdq_unregister()
286 	cmdq->db_id = 0;  in ivpu_cmdq_unregister()
299 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_destroy()  argument
301 	ivpu_cmdq_unregister(file_priv, cmdq);  in ivpu_cmdq_destroy()
302 	xa_erase(&file_priv->cmdq_xa, cmdq->id);  in ivpu_cmdq_destroy()
303 	ivpu_cmdq_free(file_priv, cmdq);  in ivpu_cmdq_destroy()
308 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_acquire_legacy()  local
313 	xa_for_each(&file_priv->cmdq_xa, id, cmdq)  in ivpu_cmdq_acquire_legacy()
314 		if (cmdq->is_legacy && cmdq->priority == priority)  in ivpu_cmdq_acquire_legacy()
317 	if (!cmdq) {  in ivpu_cmdq_acquire_legacy()
318 		cmdq = ivpu_cmdq_create(file_priv, priority, 0);  in ivpu_cmdq_acquire_legacy()
319 		if (!cmdq)  in ivpu_cmdq_acquire_legacy()
321 		cmdq->is_legacy = true;  in ivpu_cmdq_acquire_legacy()
324 	return cmdq;  in ivpu_cmdq_acquire_legacy()
330 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_acquire()  local
334 	cmdq = xa_load(&file_priv->cmdq_xa, cmdq_id);  in ivpu_cmdq_acquire()
335 	if (!cmdq) {  in ivpu_cmdq_acquire()
340 	return cmdq;  in ivpu_cmdq_acquire()
345 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_release_all_locked()  local
350 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)  in ivpu_cmdq_release_all_locked()
351 		ivpu_cmdq_destroy(file_priv, cmdq);  in ivpu_cmdq_release_all_locked()
362 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_reset()  local
367 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) {  in ivpu_cmdq_reset()
368 		xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);  in ivpu_cmdq_reset()
369 		cmdq->db_id = 0;  in ivpu_cmdq_reset()
391 	struct ivpu_cmdq *cmdq;  in ivpu_context_abort_locked()  local
397 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)  in ivpu_context_abort_locked()
398 		ivpu_cmdq_unregister(file_priv, cmdq);  in ivpu_context_abort_locked()
408 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)  in ivpu_cmdq_push_job()  argument
411 	struct vpu_job_queue_header *header = &cmdq->jobq->header;  in ivpu_cmdq_push_job()
414 	u32 next_entry = (tail + 1) % cmdq->entry_count;  in ivpu_cmdq_push_job()
418 		ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n",  in ivpu_cmdq_push_job()
419 			 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail);  in ivpu_cmdq_push_job()
423 	entry = &cmdq->jobq->slot[tail].job;  in ivpu_cmdq_push_job()
431 		if (cmdq->primary_preempt_buf) {  in ivpu_cmdq_push_job()
432 			entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;  in ivpu_cmdq_push_job()
433 			entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);  in ivpu_cmdq_push_job()
436 		if (cmdq->secondary_preempt_buf) {  in ivpu_cmdq_push_job()
437 			entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;  in ivpu_cmdq_push_job()
439 				ivpu_bo_size(cmdq->secondary_preempt_buf);  in ivpu_cmdq_push_job()
637 	struct ivpu_cmdq *cmdq;  in ivpu_job_submit()  local
649 		cmdq = ivpu_cmdq_acquire_legacy(file_priv, priority);  in ivpu_job_submit()
651 		cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id);  in ivpu_job_submit()
652 	if (!cmdq) {  in ivpu_job_submit()
658 	ret = ivpu_cmdq_register(file_priv, cmdq);  in ivpu_job_submit()
664 	job->cmdq_id = cmdq->id;  in ivpu_job_submit()
676 	ret = ivpu_cmdq_push_job(cmdq, job);  in ivpu_job_submit()
683 		cmdq->jobq->header.head = cmdq->jobq->header.tail;  in ivpu_job_submit()
686 		ivpu_cmdq_ring_db(vdev, cmdq);  in ivpu_job_submit()
693 		 job->job_id, file_priv->ctx.id, cmdq->id, job->engine_idx, cmdq->priority,  in ivpu_job_submit()
694 		 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);  in ivpu_job_submit()
904 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_create_ioctl()  local
919 	cmdq = ivpu_cmdq_create(file_priv, ivpu_job_to_jsm_priority(args->priority), args->flags);  in ivpu_cmdq_create_ioctl()
920 	if (cmdq)  in ivpu_cmdq_create_ioctl()
921 		args->cmdq_id = cmdq->id;  in ivpu_cmdq_create_ioctl()
927 	return cmdq ? 0 : -ENOMEM;  in ivpu_cmdq_create_ioctl()
935 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_destroy_ioctl()  local
948 	cmdq = xa_load(&file_priv->cmdq_xa, args->cmdq_id);  in ivpu_cmdq_destroy_ioctl()
949 	if (!cmdq || cmdq->is_legacy) {  in ivpu_cmdq_destroy_ioctl()
952 		cmdq_id = cmdq->id;  in ivpu_cmdq_destroy_ioctl()
953 		ivpu_cmdq_destroy(file_priv, cmdq);  in ivpu_cmdq_destroy_ioctl()
959 	/* Abort any pending jobs only if cmdq was destroyed */  in ivpu_cmdq_destroy_ioctl()