36 			      struct amdgpu_usermode_queue *queue,  in mes_userq_create_wptr_mapping()  argument
40 	struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;  in mes_userq_create_wptr_mapping()
42 	struct amdgpu_vm *vm = queue->vm;  in mes_userq_create_wptr_mapping()
87 	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);  in mes_userq_create_wptr_mapping()
115 static int mes_userq_map(struct amdgpu_usermode_queue *queue)  in mes_userq_map()  argument
117 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_map()
119 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;  in mes_userq_map()
120 	struct amdgpu_mqd_prop *userq_props = queue->userq_prop;  in mes_userq_map()
137 	queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);  in mes_userq_map()
139 	queue_input.process_id = queue->vm->pasid;  in mes_userq_map()
140 	queue_input.queue_type = queue->queue_type;  in mes_userq_map()
141 	queue_input.mqd_addr = queue->mqd.gpu_addr;  in mes_userq_map()
145 	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);  in mes_userq_map()
146 	queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;  in mes_userq_map()
152 		DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);  in mes_userq_map()
156 	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);  in mes_userq_map()
160 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)  in mes_userq_unmap()  argument
162 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_unmap()
165 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;  in mes_userq_unmap()
169 	queue_input.doorbell_offset = queue->doorbell_index;  in mes_userq_unmap()
176 		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);  in mes_userq_unmap()
181 				      struct amdgpu_usermode_queue *queue,  in mes_userq_create_ctx_space()  argument
184 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;  in mes_userq_create_ctx_space()
207 	struct amdgpu_usermode_queue *queue;  in mes_userq_detect_and_reset()  local
231 		xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {  in mes_userq_detect_and_reset()
232 			if (queue->queue_type == queue_type) {  in mes_userq_detect_and_reset()
234 					if (queue->doorbell_index == db_array[i]) {  in mes_userq_detect_and_reset()
235 						queue->state = AMDGPU_USERQ_STATE_HUNG;  in mes_userq_detect_and_reset()
238 						amdgpu_userq_fence_driver_force_completion(queue);  in mes_userq_detect_and_reset()
254 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,  in mes_userq_mqd_create()  argument
257 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_mqd_create()
259 	struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];  in mes_userq_mqd_create()
271 	r = amdgpu_userq_create_object(uq_mgr, &queue->mqd,  in mes_userq_mqd_create()
283 	userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;  in mes_userq_mqd_create()
285 	userq_props->doorbell_index = queue->doorbell_index;  in mes_userq_mqd_create()
286 	userq_props->fence_address = queue->fence_drv->gpu_addr;  in mes_userq_mqd_create()
288 	if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {  in mes_userq_mqd_create()
304 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);  in mes_userq_mqd_create()
309 		r = amdgpu_userq_input_va_validate(adev, queue, compute_mqd->eop_va,  in mes_userq_mqd_create()
311 		amdgpu_bo_unreserve(queue->vm->root.bo);  in mes_userq_mqd_create()
324 	} else if (queue->queue_type == AMDGPU_HW_IP_GFX) {  in mes_userq_mqd_create()
353 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);  in mes_userq_mqd_create()
358 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,  in mes_userq_mqd_create()
361 			amdgpu_bo_unreserve(queue->vm->root.bo);  in mes_userq_mqd_create()
366 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,  in mes_userq_mqd_create()
368 		amdgpu_bo_unreserve(queue->vm->root.bo);  in mes_userq_mqd_create()
375 	} else if (queue->queue_type == AMDGPU_HW_IP_DMA) {  in mes_userq_mqd_create()
391 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);  in mes_userq_mqd_create()
396 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,  in mes_userq_mqd_create()
398 		amdgpu_bo_unreserve(queue->vm->root.bo);  in mes_userq_mqd_create()
408 	queue->userq_prop = userq_props;  in mes_userq_mqd_create()
410 	r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);  in mes_userq_mqd_create()
417 	r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);  in mes_userq_mqd_create()
424 	r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);  in mes_userq_mqd_create()
433 	amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);  in mes_userq_mqd_create()
436 	amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);  in mes_userq_mqd_create()
444 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)  in mes_userq_mqd_destroy()  argument
446 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_mqd_destroy()
448 	amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);  in mes_userq_mqd_destroy()
449 	kfree(queue->userq_prop);  in mes_userq_mqd_destroy()
450 	amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);  in mes_userq_mqd_destroy()
453 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)  in mes_userq_preempt()  argument
455 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_preempt()
458 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;  in mes_userq_preempt()
465 	if (queue->state != AMDGPU_USERQ_STATE_MAPPED)  in mes_userq_preempt()
499 static int mes_userq_restore(struct amdgpu_usermode_queue *queue)  in mes_userq_restore()  argument
501 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;  in mes_userq_restore()
504 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;  in mes_userq_restore()
507 	if (queue->state == AMDGPU_USERQ_STATE_HUNG)  in mes_userq_restore()
509 	if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)  in mes_userq_restore()
519 		dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);  in mes_userq_restore()