82 			  struct amdgpu_usermode_queue *queue)  in amdgpu_userq_preempt_helper()  argument
86 		adev->userq_funcs[queue->queue_type];  in amdgpu_userq_preempt_helper()
89 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {  in amdgpu_userq_preempt_helper()
90 		r = userq_funcs->preempt(uq_mgr, queue);  in amdgpu_userq_preempt_helper()
92 			queue->state = AMDGPU_USERQ_STATE_HUNG;  in amdgpu_userq_preempt_helper()
94 			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;  in amdgpu_userq_preempt_helper()
103 			struct amdgpu_usermode_queue *queue)  in amdgpu_userq_restore_helper()  argument
107 		adev->userq_funcs[queue->queue_type];  in amdgpu_userq_restore_helper()
110 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {  in amdgpu_userq_restore_helper()
111 		r = userq_funcs->restore(uq_mgr, queue);  in amdgpu_userq_restore_helper()
113 			queue->state = AMDGPU_USERQ_STATE_HUNG;  in amdgpu_userq_restore_helper()
115 			queue->state = AMDGPU_USERQ_STATE_MAPPED;  in amdgpu_userq_restore_helper()
124 			  struct amdgpu_usermode_queue *queue)  in amdgpu_userq_unmap_helper()  argument
128 		adev->userq_funcs[queue->queue_type];  in amdgpu_userq_unmap_helper()
131 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||  in amdgpu_userq_unmap_helper()
132 		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {  in amdgpu_userq_unmap_helper()
133 		r = userq_funcs->unmap(uq_mgr, queue);  in amdgpu_userq_unmap_helper()
135 			queue->state = AMDGPU_USERQ_STATE_HUNG;  in amdgpu_userq_unmap_helper()
137 			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;  in amdgpu_userq_unmap_helper()
144 			struct amdgpu_usermode_queue *queue)  in amdgpu_userq_map_helper()  argument
148 		adev->userq_funcs[queue->queue_type];  in amdgpu_userq_map_helper()
151 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {  in amdgpu_userq_map_helper()
152 		r = userq_funcs->map(uq_mgr, queue);  in amdgpu_userq_map_helper()
154 			queue->state = AMDGPU_USERQ_STATE_HUNG;  in amdgpu_userq_map_helper()
156 			queue->state = AMDGPU_USERQ_STATE_MAPPED;  in amdgpu_userq_map_helper()
164 				 struct amdgpu_usermode_queue *queue)  in amdgpu_userq_wait_for_last_fence()  argument
166 	struct dma_fence *f = queue->last_fence;  in amdgpu_userq_wait_for_last_fence()
179 		     struct amdgpu_usermode_queue *queue,  in amdgpu_userq_cleanup()  argument
183 	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];  in amdgpu_userq_cleanup()
185 	uq_funcs->mqd_destroy(uq_mgr, queue);  in amdgpu_userq_cleanup()
186 	amdgpu_userq_fence_driver_free(queue);  in amdgpu_userq_cleanup()
188 	kfree(queue);  in amdgpu_userq_cleanup()
309 	/* Pin the BO before generating the index, unpin in queue destroy */  in amdgpu_userq_get_doorbell_index()
362 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_destroy()  local
368 	queue = amdgpu_userq_find(uq_mgr, queue_id);  in amdgpu_userq_destroy()
369 	if (!queue) {  in amdgpu_userq_destroy()
370 		drm_dbg_driver(adev_to_drm(uq_mgr->adev), "Invalid queue id to destroy\n");  in amdgpu_userq_destroy()
374 	amdgpu_userq_wait_for_last_fence(uq_mgr, queue);  in amdgpu_userq_destroy()
375 	r = amdgpu_bo_reserve(queue->db_obj.obj, true);  in amdgpu_userq_destroy()
377 		amdgpu_bo_unpin(queue->db_obj.obj);  in amdgpu_userq_destroy()
378 		amdgpu_bo_unreserve(queue->db_obj.obj);  in amdgpu_userq_destroy()
380 	amdgpu_bo_unref(&queue->db_obj.obj);  in amdgpu_userq_destroy()
383 	debugfs_remove_recursive(queue->debugfs_queue);  in amdgpu_userq_destroy()
385 	r = amdgpu_userq_unmap_helper(uq_mgr, queue);  in amdgpu_userq_destroy()
389 		queue->state = AMDGPU_USERQ_STATE_HUNG;  in amdgpu_userq_destroy()
391 	amdgpu_userq_cleanup(uq_mgr, queue, queue_id);  in amdgpu_userq_destroy()
418 	struct amdgpu_usermode_queue *queue = m->private;  in amdgpu_mqd_info_read()  local
422 	if (!queue || !queue->mqd.obj)  in amdgpu_mqd_info_read()
425 	bo = amdgpu_bo_ref(queue->mqd.obj);  in amdgpu_mqd_info_read()
432 	seq_printf(m, "queue_type: %d\n", queue->queue_type);  in amdgpu_mqd_info_read()
433 	seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));  in amdgpu_mqd_info_read()
462 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_create()  local
484 	 * There could be a situation that we are creating a new queue while  in amdgpu_userq_create()
495 		drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",  in amdgpu_userq_create()
501 	queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL);  in amdgpu_userq_create()
502 	if (!queue) {  in amdgpu_userq_create()
503 		drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");  in amdgpu_userq_create()
513 		kfree(queue);  in amdgpu_userq_create()
516 	queue->doorbell_handle = args->in.doorbell_handle;  in amdgpu_userq_create()
517 	queue->queue_type = args->in.ip_type;  in amdgpu_userq_create()
518 	queue->vm = &fpriv->vm;  in amdgpu_userq_create()
519 	queue->priority = priority;  in amdgpu_userq_create()
521 	db_info.queue_type = queue->queue_type;  in amdgpu_userq_create()
522 	db_info.doorbell_handle = queue->doorbell_handle;  in amdgpu_userq_create()
523 	db_info.db_obj = &queue->db_obj;  in amdgpu_userq_create()
529 		drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");  in amdgpu_userq_create()
530 		kfree(queue);  in amdgpu_userq_create()
535 	queue->doorbell_index = index;  in amdgpu_userq_create()
536 	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);  in amdgpu_userq_create()
537 	r = amdgpu_userq_fence_driver_alloc(adev, queue);  in amdgpu_userq_create()
543 	r = uq_funcs->mqd_create(uq_mgr, &args->in, queue);  in amdgpu_userq_create()
545 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");  in amdgpu_userq_create()
546 		amdgpu_userq_fence_driver_free(queue);  in amdgpu_userq_create()
547 		kfree(queue);  in amdgpu_userq_create()
552 	qid = idr_alloc(&uq_mgr->userq_idr, queue, 1, AMDGPU_MAX_USERQ_COUNT, GFP_KERNEL);  in amdgpu_userq_create()
554 		drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n");  in amdgpu_userq_create()
555 		amdgpu_userq_fence_driver_free(queue);  in amdgpu_userq_create()
556 		uq_funcs->mqd_destroy(uq_mgr, queue);  in amdgpu_userq_create()
557 		kfree(queue);  in amdgpu_userq_create()
562 	/* don't map the queue if scheduling is halted */  in amdgpu_userq_create()
564 	    ((queue->queue_type == AMDGPU_HW_IP_GFX) ||  in amdgpu_userq_create()
565 	     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))  in amdgpu_userq_create()
570 		r = amdgpu_userq_map_helper(uq_mgr, queue);  in amdgpu_userq_create()
572 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");  in amdgpu_userq_create()
574 			amdgpu_userq_fence_driver_free(queue);  in amdgpu_userq_create()
575 			uq_funcs->mqd_destroy(uq_mgr, queue);  in amdgpu_userq_create()
576 			kfree(queue);  in amdgpu_userq_create()
581 	queue_name = kasprintf(GFP_KERNEL, "queue-%d", qid);  in amdgpu_userq_create()
588 	/* Queue dentry per client to hold MQD information   */  in amdgpu_userq_create()
589 	queue->debugfs_queue = debugfs_create_dir(queue_name, filp->debugfs_client);  in amdgpu_userq_create()
590 	debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);  in amdgpu_userq_create()
618 			drm_file_err(filp, "Usermode queue doesn't support IP type %u\n",  in amdgpu_userq_input_args_validate()
634 			drm_file_err(filp, "invalidate userq queue va or size\n");  in amdgpu_userq_input_args_validate()
638 			drm_file_err(filp, "invalidate userq queue rptr or wptr\n");  in amdgpu_userq_input_args_validate()
675 			drm_file_err(filp, "Failed to create usermode queue\n");  in amdgpu_userq_ioctl()
681 			drm_file_err(filp, "Failed to destroy usermode queue\n");  in amdgpu_userq_ioctl()
685 		drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op);  in amdgpu_userq_ioctl()
695 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_restore_all()  local
700 	idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) {  in amdgpu_userq_restore_all()
701 		r = amdgpu_userq_restore_helper(uq_mgr, queue);  in amdgpu_userq_restore_all()
850 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_evict_all()  local
855 	idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) {  in amdgpu_userq_evict_all()
856 		r = amdgpu_userq_preempt_helper(uq_mgr, queue);  in amdgpu_userq_evict_all()
869 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_wait_for_signal()  local
872 	idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) {  in amdgpu_userq_wait_for_signal()
873 		struct dma_fence *f = queue->last_fence;  in amdgpu_userq_wait_for_signal()
940 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_mgr_fini()  local
948 	idr_for_each_entry(&userq_mgr->userq_idr, queue, queue_id) {  in amdgpu_userq_mgr_fini()
949 		amdgpu_userq_wait_for_last_fence(userq_mgr, queue);  in amdgpu_userq_mgr_fini()
950 		amdgpu_userq_unmap_helper(userq_mgr, queue);  in amdgpu_userq_mgr_fini()
951 		amdgpu_userq_cleanup(userq_mgr, queue, queue_id);  in amdgpu_userq_mgr_fini()
969 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_suspend()  local
981 		idr_for_each_entry(&uqm->userq_idr, queue, queue_id) {  in amdgpu_userq_suspend()
983 				r = amdgpu_userq_preempt_helper(uqm, queue);  in amdgpu_userq_suspend()
985 				r = amdgpu_userq_unmap_helper(uqm, queue);  in amdgpu_userq_suspend()
998 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_resume()  local
1009 		idr_for_each_entry(&uqm->userq_idr, queue, queue_id) {  in amdgpu_userq_resume()
1011 				r = amdgpu_userq_restore_helper(uqm, queue);  in amdgpu_userq_resume()
1013 				r = amdgpu_userq_map_helper(uqm, queue);  in amdgpu_userq_resume()
1027 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_stop_sched_for_enforce_isolation()  local
1043 		idr_for_each_entry(&uqm->userq_idr, queue, queue_id) {  in amdgpu_userq_stop_sched_for_enforce_isolation()
1044 			if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||  in amdgpu_userq_stop_sched_for_enforce_isolation()
1045 			     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&  in amdgpu_userq_stop_sched_for_enforce_isolation()
1046 			    (queue->xcp_id == idx)) {  in amdgpu_userq_stop_sched_for_enforce_isolation()
1047 				r = amdgpu_userq_preempt_helper(uqm, queue);  in amdgpu_userq_stop_sched_for_enforce_isolation()
1062 	struct amdgpu_usermode_queue *queue;  in amdgpu_userq_start_sched_for_enforce_isolation()  local
1077 		idr_for_each_entry(&uqm->userq_idr, queue, queue_id) {  in amdgpu_userq_start_sched_for_enforce_isolation()
1078 			if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||  in amdgpu_userq_start_sched_for_enforce_isolation()
1079 			     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&  in amdgpu_userq_start_sched_for_enforce_isolation()
1080 			    (queue->xcp_id == idx)) {  in amdgpu_userq_start_sched_for_enforce_isolation()
1081 				r = amdgpu_userq_restore_helper(uqm, queue);  in amdgpu_userq_start_sched_for_enforce_isolation()