116 	pvr_context_put(fence->queue->ctx);  in pvr_queue_fence_release_work()
123 	struct pvr_device *pvr_dev = fence->queue->ctx->pvr_dev;  in pvr_queue_fence_release()
133 	switch (fence->queue->type) {  in pvr_queue_job_fence_get_timeline_name()
156 	switch (fence->queue->type) {  in pvr_queue_cccb_fence_get_timeline_name()
271 		     struct pvr_queue *queue,  in pvr_queue_fence_init()  argument
277 	pvr_context_get(queue->ctx);  in pvr_queue_fence_init()
278 	fence->queue = queue;  in pvr_queue_fence_init()
296 pvr_queue_cccb_fence_init(struct dma_fence *fence, struct pvr_queue *queue)  in pvr_queue_cccb_fence_init()  argument
298 	pvr_queue_fence_init(fence, queue, &pvr_queue_cccb_fence_ops,  in pvr_queue_cccb_fence_init()
299 			     &queue->cccb_fence_ctx.base);  in pvr_queue_cccb_fence_init()
314 pvr_queue_job_fence_init(struct dma_fence *fence, struct pvr_queue *queue)  in pvr_queue_job_fence_init()  argument
317 		pvr_queue_fence_init(fence, queue, &pvr_queue_job_fence_ops,  in pvr_queue_job_fence_init()
318 				     &queue->job_fence_ctx);  in pvr_queue_job_fence_init()
398 pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_job_cccb_fence()  argument
409 	mutex_lock(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_get_job_cccb_fence()
413 	if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {  in pvr_queue_get_job_cccb_fence()
422 	if (WARN_ON(queue->cccb_fence_ctx.job))  in pvr_queue_get_job_cccb_fence()
423 		pvr_job_put(queue->cccb_fence_ctx.job);  in pvr_queue_get_job_cccb_fence()
425 	queue->cccb_fence_ctx.job = pvr_job_get(job);  in pvr_queue_get_job_cccb_fence()
429 	if (!WARN_ON(cccb_fence->queue))  in pvr_queue_get_job_cccb_fence()
430 		pvr_queue_cccb_fence_init(job->cccb_fence, queue);  in pvr_queue_get_job_cccb_fence()
433 	mutex_unlock(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_get_job_cccb_fence()
451 pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_job_kccb_fence()  argument
453 	struct pvr_device *pvr_dev = queue->ctx->pvr_dev;  in pvr_queue_get_job_kccb_fence()
471 pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)  in pvr_queue_get_paired_frag_job_dep()  argument
493 	return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity);  in pvr_queue_get_paired_frag_job_dep()
509 	struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);  in pvr_queue_prepare_job()  local
535 		pvr_queue_job_fence_init(job->done_fence, queue);  in pvr_queue_prepare_job()
541 	internal_dep = pvr_queue_get_job_cccb_fence(queue, job);  in pvr_queue_prepare_job()
547 		internal_dep = pvr_queue_get_job_kccb_fence(queue, job);  in pvr_queue_prepare_job()
558 		internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job);  in pvr_queue_prepare_job()
570 static void pvr_queue_update_active_state_locked(struct pvr_queue *queue)  in pvr_queue_update_active_state_locked()  argument
572 	struct pvr_device *pvr_dev = queue->ctx->pvr_dev;  in pvr_queue_update_active_state_locked()
580 	if (list_empty(&queue->node))  in pvr_queue_update_active_state_locked()
583 	if (!atomic_read(&queue->in_flight_job_count))  in pvr_queue_update_active_state_locked()
584 		list_move_tail(&queue->node, &pvr_dev->queues.idle);  in pvr_queue_update_active_state_locked()
586 		list_move_tail(&queue->node, &pvr_dev->queues.active);  in pvr_queue_update_active_state_locked()
602 static void pvr_queue_update_active_state(struct pvr_queue *queue)  in pvr_queue_update_active_state()  argument
604 	struct pvr_device *pvr_dev = queue->ctx->pvr_dev;  in pvr_queue_update_active_state()
607 	pvr_queue_update_active_state_locked(queue);  in pvr_queue_update_active_state()
613 	struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);  in pvr_queue_submit_job_to_cccb()  local
615 	struct pvr_cccb *cccb = &queue->cccb;  in pvr_queue_submit_job_to_cccb()
625 	atomic_inc(&queue->in_flight_job_count);  in pvr_queue_submit_job_to_cccb()
626 	pvr_queue_update_active_state(queue);  in pvr_queue_submit_job_to_cccb()
641 		pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,  in pvr_queue_submit_job_to_cccb()
656 			pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,  in pvr_queue_submit_job_to_cccb()
673 		pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj,  in pvr_queue_submit_job_to_cccb()
683 	pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj, &ufos[0].addr);  in pvr_queue_submit_job_to_cccb()
755 		struct pvr_queue *queue = container_of(job->base.sched,  in pvr_queue_run_job()  local
758 		pvr_cccb_send_kccb_kick(pvr_dev, &queue->cccb,  in pvr_queue_run_job()
759 					pvr_context_get_fw_addr(job->ctx) + queue->ctx_offset,  in pvr_queue_run_job()
766 static void pvr_queue_stop(struct pvr_queue *queue, struct pvr_job *bad_job)  in pvr_queue_stop()  argument
768 	drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);  in pvr_queue_stop()
771 static void pvr_queue_start(struct pvr_queue *queue)  in pvr_queue_start()  argument
778 	*queue->timeline_ufo.value = atomic_read(&queue->job_fence_ctx.seqno);  in pvr_queue_start()
780 	list_for_each_entry(job, &queue->scheduler.pending_list, base.list) {  in pvr_queue_start()
791 			atomic_set(&queue->ctx->faulty, 1);  in pvr_queue_start()
795 	drm_sched_start(&queue->scheduler, 0);  in pvr_queue_start()
812 	struct pvr_queue *queue = container_of(sched, struct pvr_queue, scheduler);  in pvr_queue_timedout_job()  local
813 	struct pvr_device *pvr_dev = queue->ctx->pvr_dev;  in pvr_queue_timedout_job()
830 	list_del_init(&queue->node);  in pvr_queue_timedout_job()
840 	WARN_ON(atomic_read(&queue->in_flight_job_count) != job_count);  in pvr_queue_timedout_job()
847 		list_move_tail(&queue->node, &pvr_dev->queues.idle);  in pvr_queue_timedout_job()
849 		atomic_set(&queue->in_flight_job_count, job_count);  in pvr_queue_timedout_job()
850 		list_move_tail(&queue->node, &pvr_dev->queues.active);  in pvr_queue_timedout_job()
851 		pvr_queue_process(queue);  in pvr_queue_timedout_job()
916 pvr_queue_signal_done_fences(struct pvr_queue *queue)  in pvr_queue_signal_done_fences()  argument
921 	spin_lock(&queue->scheduler.job_list_lock);  in pvr_queue_signal_done_fences()
922 	cur_seqno = *queue->timeline_ufo.value;  in pvr_queue_signal_done_fences()
923 	list_for_each_entry_safe(job, tmp_job, &queue->scheduler.pending_list, base.list) {  in pvr_queue_signal_done_fences()
930 			atomic_dec(&queue->in_flight_job_count);  in pvr_queue_signal_done_fences()
933 	spin_unlock(&queue->scheduler.job_list_lock);  in pvr_queue_signal_done_fences()
946 pvr_queue_check_job_waiting_for_cccb_space(struct pvr_queue *queue)  in pvr_queue_check_job_waiting_for_cccb_space()  argument
952 	mutex_lock(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_check_job_waiting_for_cccb_space()
953 	job = queue->cccb_fence_ctx.job;  in pvr_queue_check_job_waiting_for_cccb_space()
967 	if (WARN_ON(!cccb_fence->queue)) {  in pvr_queue_check_job_waiting_for_cccb_space()
977 	if (!pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {  in pvr_queue_check_job_waiting_for_cccb_space()
985 	queue->cccb_fence_ctx.job = NULL;  in pvr_queue_check_job_waiting_for_cccb_space()
988 	mutex_unlock(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_check_job_waiting_for_cccb_space()
999 void pvr_queue_process(struct pvr_queue *queue)  in pvr_queue_process()  argument
1001 	lockdep_assert_held(&queue->ctx->pvr_dev->queues.lock);  in pvr_queue_process()
1003 	pvr_queue_check_job_waiting_for_cccb_space(queue);  in pvr_queue_process()
1004 	pvr_queue_signal_done_fences(queue);  in pvr_queue_process()
1005 	pvr_queue_update_active_state_locked(queue);  in pvr_queue_process()
1008 static u32 get_dm_type(struct pvr_queue *queue)  in get_dm_type()  argument
1010 	switch (queue->type) {  in get_dm_type()
1032 static void init_fw_context(struct pvr_queue *queue, void *fw_ctx_map)  in init_fw_context()  argument
1034 	struct pvr_context *ctx = queue->ctx;  in init_fw_context()
1037 	struct pvr_cccb *cccb = &queue->cccb;  in init_fw_context()
1039 	cctx_fw = fw_ctx_map + queue->ctx_offset;  in init_fw_context()
1043 	cctx_fw->dm = get_dm_type(queue);  in init_fw_context()
1052 	pvr_fw_object_get_fw_addr(queue->reg_state_obj, &cctx_fw->context_state_addr);  in init_fw_context()
1063 static int pvr_queue_cleanup_fw_context(struct pvr_queue *queue)  in pvr_queue_cleanup_fw_context()  argument
1065 	if (!queue->ctx->fw_obj)  in pvr_queue_cleanup_fw_context()
1068 	return pvr_fw_structure_cleanup(queue->ctx->pvr_dev,  in pvr_queue_cleanup_fw_context()
1070 					queue->ctx->fw_obj, queue->ctx_offset);  in pvr_queue_cleanup_fw_context()
1090 	struct pvr_queue *queue;  in pvr_queue_job_init()  local
1096 	queue = pvr_context_get_queue_for_job(job->ctx, job->type);  in pvr_queue_job_init()
1097 	if (!queue)  in pvr_queue_job_init()
1100 	if (!pvr_cccb_cmdseq_can_fit(&queue->cccb, job_cmds_size(job, min_native_dep_count)))  in pvr_queue_job_init()
1103 	err = drm_sched_job_init(&job->base, &queue->entity, 1, THIS_MODULE, drm_client_id);  in pvr_queue_job_init()
1166 	struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);  in pvr_queue_job_push()  local
1169 	dma_fence_put(queue->last_queued_job_scheduled_fence);  in pvr_queue_job_push()
1170 	queue->last_queued_job_scheduled_fence = dma_fence_get(&job->base.s_fence->scheduled);  in pvr_queue_job_push()
1178 	struct pvr_queue *queue = priv;  in reg_state_init()  local
1180 	if (queue->type == DRM_PVR_JOB_TYPE_GEOMETRY) {  in reg_state_init()
1184 			queue->callstack_addr;  in reg_state_init()
1240 	struct pvr_queue *queue;  in pvr_queue_create()  local
1269 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);  in pvr_queue_create()
1270 	if (!queue)  in pvr_queue_create()
1273 	queue->type = type;  in pvr_queue_create()
1274 	queue->ctx_offset = get_ctx_offset(type);  in pvr_queue_create()
1275 	queue->ctx = ctx;  in pvr_queue_create()
1276 	queue->callstack_addr = args->callstack_addr;  in pvr_queue_create()
1277 	sched = &queue->scheduler;  in pvr_queue_create()
1278 	INIT_LIST_HEAD(&queue->node);  in pvr_queue_create()
1279 	mutex_init(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_create()
1280 	pvr_queue_fence_ctx_init(&queue->cccb_fence_ctx.base);  in pvr_queue_create()
1281 	pvr_queue_fence_ctx_init(&queue->job_fence_ctx);  in pvr_queue_create()
1283 	err = pvr_cccb_init(pvr_dev, &queue->cccb, props[type].cccb_size, props[type].name);  in pvr_queue_create()
1289 				   reg_state_init, queue, &queue->reg_state_obj);  in pvr_queue_create()
1293 	init_fw_context(queue, fw_ctx_map);  in pvr_queue_create()
1301 	cpu_map = pvr_fw_object_create_and_map(pvr_dev, sizeof(*queue->timeline_ufo.value),  in pvr_queue_create()
1303 					       NULL, NULL, &queue->timeline_ufo.fw_obj);  in pvr_queue_create()
1309 	queue->timeline_ufo.value = cpu_map;  in pvr_queue_create()
1311 	err = drm_sched_init(&queue->scheduler, &sched_args);  in pvr_queue_create()
1315 	err = drm_sched_entity_init(&queue->entity,  in pvr_queue_create()
1322 	list_add_tail(&queue->node, &pvr_dev->queues.idle);  in pvr_queue_create()
1325 	return queue;  in pvr_queue_create()
1328 	drm_sched_fini(&queue->scheduler);  in pvr_queue_create()
1331 	pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);  in pvr_queue_create()
1334 	pvr_fw_object_destroy(queue->reg_state_obj);  in pvr_queue_create()
1337 	pvr_cccb_fini(&queue->cccb);  in pvr_queue_create()
1340 	mutex_destroy(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_create()
1341 	kfree(queue);  in pvr_queue_create()
1348 	struct pvr_queue *queue;  in pvr_queue_device_pre_reset()  local
1351 	list_for_each_entry(queue, &pvr_dev->queues.idle, node)  in pvr_queue_device_pre_reset()
1352 		pvr_queue_stop(queue, NULL);  in pvr_queue_device_pre_reset()
1353 	list_for_each_entry(queue, &pvr_dev->queues.active, node)  in pvr_queue_device_pre_reset()
1354 		pvr_queue_stop(queue, NULL);  in pvr_queue_device_pre_reset()
1360 	struct pvr_queue *queue;  in pvr_queue_device_post_reset()  local
1363 	list_for_each_entry(queue, &pvr_dev->queues.active, node)  in pvr_queue_device_post_reset()
1364 		pvr_queue_start(queue);  in pvr_queue_device_post_reset()
1365 	list_for_each_entry(queue, &pvr_dev->queues.idle, node)  in pvr_queue_device_post_reset()
1366 		pvr_queue_start(queue);  in pvr_queue_device_post_reset()
1379 void pvr_queue_kill(struct pvr_queue *queue)  in pvr_queue_kill()  argument
1381 	drm_sched_entity_destroy(&queue->entity);  in pvr_queue_kill()
1382 	dma_fence_put(queue->last_queued_job_scheduled_fence);  in pvr_queue_kill()
1383 	queue->last_queued_job_scheduled_fence = NULL;  in pvr_queue_kill()
1393 void pvr_queue_destroy(struct pvr_queue *queue)  in pvr_queue_destroy()  argument
1395 	if (!queue)  in pvr_queue_destroy()
1398 	mutex_lock(&queue->ctx->pvr_dev->queues.lock);  in pvr_queue_destroy()
1399 	list_del_init(&queue->node);  in pvr_queue_destroy()
1400 	mutex_unlock(&queue->ctx->pvr_dev->queues.lock);  in pvr_queue_destroy()
1402 	drm_sched_fini(&queue->scheduler);  in pvr_queue_destroy()
1403 	drm_sched_entity_fini(&queue->entity);  in pvr_queue_destroy()
1405 	if (WARN_ON(queue->last_queued_job_scheduled_fence))  in pvr_queue_destroy()
1406 		dma_fence_put(queue->last_queued_job_scheduled_fence);  in pvr_queue_destroy()
1408 	pvr_queue_cleanup_fw_context(queue);  in pvr_queue_destroy()
1410 	pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);  in pvr_queue_destroy()
1411 	pvr_fw_object_destroy(queue->reg_state_obj);  in pvr_queue_destroy()
1412 	pvr_cccb_fini(&queue->cccb);  in pvr_queue_destroy()
1413 	mutex_destroy(&queue->cccb_fence_ctx.job_lock);  in pvr_queue_destroy()
1414 	kfree(queue);  in pvr_queue_destroy()