350 static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count)  in job_cmds_size()  argument
356 	       pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len);  in job_cmds_size()
365 static unsigned long job_count_remaining_native_deps(struct pvr_job *job)  in job_count_remaining_native_deps()  argument
371 	xa_for_each(&job->base.dependencies, index, fence) {  in job_count_remaining_native_deps()
398 pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_job_cccb_fence()  argument
406 	if (!job->cccb_fence)  in pvr_queue_get_job_cccb_fence()
412 	native_deps_remaining = job_count_remaining_native_deps(job);  in pvr_queue_get_job_cccb_fence()
413 	if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {  in pvr_queue_get_job_cccb_fence()
414 		pvr_queue_fence_put(job->cccb_fence);  in pvr_queue_get_job_cccb_fence()
415 		job->cccb_fence = NULL;  in pvr_queue_get_job_cccb_fence()
422 	if (WARN_ON(queue->cccb_fence_ctx.job))  in pvr_queue_get_job_cccb_fence()
423 		pvr_job_put(queue->cccb_fence_ctx.job);  in pvr_queue_get_job_cccb_fence()
425 	queue->cccb_fence_ctx.job = pvr_job_get(job);  in pvr_queue_get_job_cccb_fence()
428 	cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);  in pvr_queue_get_job_cccb_fence()
430 		pvr_queue_cccb_fence_init(job->cccb_fence, queue);  in pvr_queue_get_job_cccb_fence()
435 	return dma_fence_get(job->cccb_fence);  in pvr_queue_get_job_cccb_fence()
451 pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_job_kccb_fence()  argument
459 	if (!job->kccb_fence)  in pvr_queue_get_job_kccb_fence()
462 	if (!WARN_ON(job->kccb_fence->ops)) {  in pvr_queue_get_job_kccb_fence()
463 		kccb_fence = pvr_kccb_reserve_slot(pvr_dev, job->kccb_fence);  in pvr_queue_get_job_kccb_fence()
464 		job->kccb_fence = NULL;  in pvr_queue_get_job_kccb_fence()
471 pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_paired_frag_job_dep()  argument
473 	struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?  in pvr_queue_get_paired_frag_job_dep()
474 				   job->paired_job : NULL;  in pvr_queue_get_paired_frag_job_dep()
487 		if (f == &job->base.s_fence->scheduled)  in pvr_queue_get_paired_frag_job_dep()
508 	struct pvr_job *job = container_of(sched_job, struct pvr_job, base);  in pvr_queue_prepare_job()  local
517 	if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {  in pvr_queue_prepare_job()
524 		if (job->paired_job->has_pm_ref)  in pvr_queue_prepare_job()
532 		pvr_queue_job_fence_init(job->done_fence,  in pvr_queue_prepare_job()
533 					 job->ctx->queues.fragment);  in pvr_queue_prepare_job()
535 		pvr_queue_job_fence_init(job->done_fence, queue);  in pvr_queue_prepare_job()
541 	internal_dep = pvr_queue_get_job_cccb_fence(queue, job);  in pvr_queue_prepare_job()
547 		internal_dep = pvr_queue_get_job_kccb_fence(queue, job);  in pvr_queue_prepare_job()
558 		internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job);  in pvr_queue_prepare_job()
611 static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)  in pvr_queue_submit_job_to_cccb()  argument
613 	struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);  in pvr_queue_submit_job_to_cccb()
628 	xa_for_each(&job->base.dependencies, index, fence) {  in pvr_queue_submit_job_to_cccb()
634 		if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job &&  in pvr_queue_submit_job_to_cccb()
635 		    &job->paired_job->base.s_fence->scheduled == fence)  in pvr_queue_submit_job_to_cccb()
653 	if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {  in pvr_queue_submit_job_to_cccb()
654 		jfence = to_pvr_queue_job_fence(job->paired_job->done_fence);  in pvr_queue_submit_job_to_cccb()
658 			ufos[ufo_count++].value = job->paired_job->done_fence->seqno;  in pvr_queue_submit_job_to_cccb()
667 	if (job->type == DRM_PVR_JOB_TYPE_GEOMETRY && job->paired_job) {  in pvr_queue_submit_job_to_cccb()
668 		struct rogue_fwif_cmd_geom *cmd = job->cmd;  in pvr_queue_submit_job_to_cccb()
675 		cmd->partial_render_geom_frag_fence.value = job->done_fence->seqno - 1;  in pvr_queue_submit_job_to_cccb()
679 	pvr_cccb_write_command_with_header(cccb, job->fw_ccb_cmd_type, job->cmd_len, job->cmd,  in pvr_queue_submit_job_to_cccb()
680 					   job->id, job->id);  in pvr_queue_submit_job_to_cccb()
684 	ufos[0].value = job->done_fence->seqno;  in pvr_queue_submit_job_to_cccb()
698 	struct pvr_job *job = container_of(sched_job, struct pvr_job, base);  in pvr_queue_run_job()  local
699 	struct pvr_device *pvr_dev = job->pvr_dev;  in pvr_queue_run_job()
706 	if (job->paired_job && job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&  in pvr_queue_run_job()
707 	    job->done_fence->ops) {  in pvr_queue_run_job()
708 		return dma_fence_get(job->done_fence);  in pvr_queue_run_job()
717 	if (WARN_ON(job->paired_job &&  in pvr_queue_run_job()
718 		    (job->type != DRM_PVR_JOB_TYPE_GEOMETRY ||  in pvr_queue_run_job()
719 		     job->paired_job->type != DRM_PVR_JOB_TYPE_FRAGMENT ||  in pvr_queue_run_job()
720 		     job->hwrt != job->paired_job->hwrt ||  in pvr_queue_run_job()
721 		     job->ctx != job->paired_job->ctx)))  in pvr_queue_run_job()
724 	err = pvr_job_get_pm_ref(job);  in pvr_queue_run_job()
728 	if (job->paired_job) {  in pvr_queue_run_job()
729 		err = pvr_job_get_pm_ref(job->paired_job);  in pvr_queue_run_job()
735 	pvr_queue_submit_job_to_cccb(job);  in pvr_queue_run_job()
737 	if (job->paired_job) {  in pvr_queue_run_job()
738 		struct pvr_job *geom_job = job;  in pvr_queue_run_job()
739 		struct pvr_job *frag_job = job->paired_job;  in pvr_queue_run_job()
740 		struct pvr_queue *geom_queue = job->ctx->queues.geometry;  in pvr_queue_run_job()
741 		struct pvr_queue *frag_queue = job->ctx->queues.fragment;  in pvr_queue_run_job()
751 						 job->hwrt,  in pvr_queue_run_job()
755 		struct pvr_queue *queue = container_of(job->base.sched,  in pvr_queue_run_job()
759 					pvr_context_get_fw_addr(job->ctx) + queue->ctx_offset,  in pvr_queue_run_job()
760 					job->hwrt);  in pvr_queue_run_job()
763 	return dma_fence_get(job->done_fence);  in pvr_queue_run_job()
773 	struct pvr_job *job;  in pvr_queue_start()  local
780 	list_for_each_entry(job, &queue->scheduler.pending_list, base.list) {  in pvr_queue_start()
781 		if (dma_fence_is_signaled(job->done_fence)) {  in pvr_queue_start()
785 			WARN_ON(job->base.s_fence->parent);  in pvr_queue_start()
786 			job->base.s_fence->parent = dma_fence_get(job->done_fence);  in pvr_queue_start()
814 	struct pvr_job *job;  in pvr_queue_timedout_job()  local
836 	list_for_each_entry(job, &sched->pending_list, base.list) {  in pvr_queue_timedout_job()
837 		job->base.s_fence->parent = dma_fence_get(job->done_fence);  in pvr_queue_timedout_job()
866 	struct pvr_job *job = container_of(sched_job, struct pvr_job, base);  in pvr_queue_free_job()  local
870 	if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job)  in pvr_queue_free_job()
871 		pvr_job_put(job->paired_job);  in pvr_queue_free_job()
873 	job->paired_job = NULL;  in pvr_queue_free_job()
874 	pvr_job_put(job);  in pvr_queue_free_job()
918 	struct pvr_job *job, *tmp_job;  in pvr_queue_signal_done_fences()  local
923 	list_for_each_entry_safe(job, tmp_job, &queue->scheduler.pending_list, base.list) {  in pvr_queue_signal_done_fences()
924 		if ((int)(cur_seqno - lower_32_bits(job->done_fence->seqno)) < 0)  in pvr_queue_signal_done_fences()
927 		if (!dma_fence_is_signaled(job->done_fence)) {  in pvr_queue_signal_done_fences()
928 			dma_fence_signal(job->done_fence);  in pvr_queue_signal_done_fences()
929 			pvr_job_release_pm_ref(job);  in pvr_queue_signal_done_fences()
950 	struct pvr_job *job;  in pvr_queue_check_job_waiting_for_cccb_space()  local
953 	job = queue->cccb_fence_ctx.job;  in pvr_queue_check_job_waiting_for_cccb_space()
954 	if (!job)  in pvr_queue_check_job_waiting_for_cccb_space()
960 	if (WARN_ON(!job->cccb_fence)) {  in pvr_queue_check_job_waiting_for_cccb_space()
961 		job = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
966 	cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);  in pvr_queue_check_job_waiting_for_cccb_space()
968 		job = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
976 	native_deps_remaining = job_count_remaining_native_deps(job);  in pvr_queue_check_job_waiting_for_cccb_space()
977 	if (!pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {  in pvr_queue_check_job_waiting_for_cccb_space()
978 		job = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
982 	dma_fence_signal(job->cccb_fence);  in pvr_queue_check_job_waiting_for_cccb_space()
983 	pvr_queue_fence_put(job->cccb_fence);  in pvr_queue_check_job_waiting_for_cccb_space()
984 	job->cccb_fence = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
985 	queue->cccb_fence_ctx.job = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
990 	pvr_job_put(job);  in pvr_queue_check_job_waiting_for_cccb_space()
1086 int pvr_queue_job_init(struct pvr_job *job, u64 drm_client_id)  in pvr_queue_job_init()  argument
1089 	u32 min_native_dep_count = job->type == DRM_PVR_JOB_TYPE_FRAGMENT ? 1 : 0;  in pvr_queue_job_init()
1093 	if (atomic_read(&job->ctx->faulty))  in pvr_queue_job_init()
1096 	queue = pvr_context_get_queue_for_job(job->ctx, job->type);  in pvr_queue_job_init()
1100 	if (!pvr_cccb_cmdseq_can_fit(&queue->cccb, job_cmds_size(job, min_native_dep_count)))  in pvr_queue_job_init()
1103 	err = drm_sched_job_init(&job->base, &queue->entity, 1, THIS_MODULE, drm_client_id);  in pvr_queue_job_init()
1107 	job->cccb_fence = pvr_queue_fence_alloc();  in pvr_queue_job_init()
1108 	job->kccb_fence = pvr_kccb_fence_alloc();  in pvr_queue_job_init()
1109 	job->done_fence = pvr_queue_fence_alloc();  in pvr_queue_job_init()
1110 	if (!job->cccb_fence || !job->kccb_fence || !job->done_fence)  in pvr_queue_job_init()
1131 struct dma_fence *pvr_queue_job_arm(struct pvr_job *job)  in pvr_queue_job_arm()  argument
1133 	drm_sched_job_arm(&job->base);  in pvr_queue_job_arm()
1135 	return &job->base.s_fence->finished;  in pvr_queue_job_arm()
1144 void pvr_queue_job_cleanup(struct pvr_job *job)  in pvr_queue_job_cleanup()  argument
1146 	pvr_queue_fence_put(job->done_fence);  in pvr_queue_job_cleanup()
1147 	pvr_queue_fence_put(job->cccb_fence);  in pvr_queue_job_cleanup()
1148 	pvr_kccb_fence_put(job->kccb_fence);  in pvr_queue_job_cleanup()
1150 	if (job->base.s_fence)  in pvr_queue_job_cleanup()
1151 		drm_sched_job_cleanup(&job->base);  in pvr_queue_job_cleanup()
1164 void pvr_queue_job_push(struct pvr_job *job)  in pvr_queue_job_push()  argument
1166 	struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);  in pvr_queue_job_push()
1170 	queue->last_queued_job_scheduled_fence = dma_fence_get(&job->base.s_fence->scheduled);  in pvr_queue_job_push()
1172 	pvr_job_get(job);  in pvr_queue_job_push()
1173 	drm_sched_entity_push_job(&job->base);  in pvr_queue_job_push()