46 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_kernel_doorbell_get()  local
53 	found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);  in amdgpu_mes_kernel_doorbell_get()
54 	if (found >= mes->num_mes_dbs) {  in amdgpu_mes_kernel_doorbell_get()
59 	set_bit(found, mes->doorbell_bitmap);  in amdgpu_mes_kernel_doorbell_get()
62 	*doorbell_index = mes->db_start_dw_offset + found * 2;  in amdgpu_mes_kernel_doorbell_get()
70 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_kernel_doorbell_free()  local
73 	rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;  in amdgpu_mes_kernel_doorbell_free()
74 	old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);  in amdgpu_mes_kernel_doorbell_free()
81 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_doorbell_init()  local
84 	mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);  in amdgpu_mes_doorbell_init()
85 	if (!mes->doorbell_bitmap) {  in amdgpu_mes_doorbell_init()
90 	mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;  in amdgpu_mes_doorbell_init()
92 		adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;  in amdgpu_mes_doorbell_init()
93 		set_bit(i, mes->doorbell_bitmap);  in amdgpu_mes_doorbell_init()
106 	r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE,  in amdgpu_mes_event_log_init()
108 				    &adev->mes.event_log_gpu_obj,  in amdgpu_mes_event_log_init()
109 				    &adev->mes.event_log_gpu_addr,  in amdgpu_mes_event_log_init()
110 				    &adev->mes.event_log_cpu_addr);  in amdgpu_mes_event_log_init()
116 	memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size);  in amdgpu_mes_event_log_init()
124 	bitmap_free(adev->mes.doorbell_bitmap);  in amdgpu_mes_doorbell_free()
131 	adev->mes.adev = adev;  in amdgpu_mes_init()
133 	idr_init(&adev->mes.pasid_idr);  in amdgpu_mes_init()
134 	idr_init(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
135 	idr_init(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
136 	ida_init(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
137 	spin_lock_init(&adev->mes.queue_id_lock);  in amdgpu_mes_init()
138 	mutex_init(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
141 		spin_lock_init(&adev->mes.ring_lock[i]);  in amdgpu_mes_init()
143 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;  in amdgpu_mes_init()
144 	adev->mes.vmid_mask_mmhub = 0xffffff00;  in amdgpu_mes_init()
145 	adev->mes.vmid_mask_gfxhub = 0xffffff00;  in amdgpu_mes_init()
151 		adev->mes.compute_hqd_mask[i] = 0xc;  in amdgpu_mes_init()
155 		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;  in amdgpu_mes_init()
160 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;  in amdgpu_mes_init()
163 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;  in amdgpu_mes_init()
165 			adev->mes.sdma_hqd_mask[i] = 0xfc;  in amdgpu_mes_init()
169 		r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_init()
176 		adev->mes.sch_ctx_gpu_addr[i] =  in amdgpu_mes_init()
177 			adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4);  in amdgpu_mes_init()
178 		adev->mes.sch_ctx_ptr[i] =  in amdgpu_mes_init()
179 			(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]];  in amdgpu_mes_init()
182 				 &adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_init()
189 		adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr +  in amdgpu_mes_init()
190 			(adev->mes.query_status_fence_offs[i] * 4);  in amdgpu_mes_init()
191 		adev->mes.query_status_fence_ptr[i] =  in amdgpu_mes_init()
192 			(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]];  in amdgpu_mes_init()
209 		if (adev->mes.sch_ctx_ptr[i])  in amdgpu_mes_init()
210 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_init()
211 		if (adev->mes.query_status_fence_ptr[i])  in amdgpu_mes_init()
213 				      adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_init()
216 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_init()
217 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
218 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
219 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
220 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
228 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,  in amdgpu_mes_fini()
229 			      &adev->mes.event_log_gpu_addr,  in amdgpu_mes_fini()
230 			      &adev->mes.event_log_cpu_addr);  in amdgpu_mes_fini()
233 		if (adev->mes.sch_ctx_ptr[i])  in amdgpu_mes_fini()
234 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_fini()
235 		if (adev->mes.query_status_fence_ptr[i])  in amdgpu_mes_fini()
237 				      adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_fini()
242 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_fini()
243 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_fini()
244 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_fini()
245 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_fini()
246 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_fini()
285 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_create_process()
288 	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,  in amdgpu_mes_create_process()
298 	process->process_quantum = adev->mes.default_process_quantum;  in amdgpu_mes_create_process()
301 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
305 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
327 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_destroy_process()
329 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
332 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
339 			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
340 			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_destroy_process()
341 			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
346 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,  in amdgpu_mes_destroy_process()
352 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_destroy_process()
355 	idr_remove(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
356 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
409 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_gang()
411 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_add_gang()
419 	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,  in amdgpu_mes_add_gang()
433 		gprops->gang_quantum : adev->mes.default_gang_quantum;  in amdgpu_mes_add_gang()
438 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
442 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
459 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_gang()
461 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_remove_gang()
464 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
470 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
474 	idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_remove_gang()
476 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
502 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_suspend()
503 	r = adev->mes.funcs->suspend_gang(&adev->mes, &input);  in amdgpu_mes_suspend()
504 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_suspend()
526 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_resume()
527 	r = adev->mes.funcs->resume_gang(&adev->mes, &input);  in amdgpu_mes_resume()
528 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_resume()
630 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_hw_queue()
632 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_hw_queue()
640 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
641 	r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,  in amdgpu_mes_add_hw_queue()
644 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
647 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
684 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_add_hw_queue()
705 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
711 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
712 	idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_add_hw_queue()
713 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
715 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
734 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
737 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
739 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
741 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
742 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
747 	idr_remove(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
748 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
757 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_remove_hw_queue()
764 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
783 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
786 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
788 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_reset_hw_queue()
790 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
791 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
795 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
804 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_hw_queue()
809 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
826 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_hw_queue_mmio()
848 	r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_map_legacy_queue()
871 	r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_unmap_legacy_queue()
898 	r = adev->mes.funcs->reset_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_legacy_queue()
923 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_rreg()
928 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_rreg()
950 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_wreg()
956 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_wreg()
977 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_write_reg_wait()
983 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_write_reg_wait()
1002 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_wait()
1008 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_wait()
1026 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_set_shader_debugger()
1043 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>  in amdgpu_mes_set_shader_debugger()
1047 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_set_shader_debugger()
1049 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_set_shader_debugger()
1053 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_set_shader_debugger()
1064 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_flush_shader_debugger()
1073 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_flush_shader_debugger()
1075 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_flush_shader_debugger()
1079 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_flush_shader_debugger()
1153 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_ring()
1154 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_ring()
1157 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1164 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1203 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1211 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1256 	return adev->mes.aggregated_doorbells[prio];  in amdgpu_mes_get_aggregated_doorbell_index()
1429 	gprops.gang_quantum = adev->mes.default_gang_quantum;  in amdgpu_mes_test_create_gang_and_queues()
1613 	r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED,  in amdgpu_mes_init_microcode()
1617 		r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],  in amdgpu_mes_init_microcode()
1626 		adev->mes.fw[pipe]->data;  in amdgpu_mes_init_microcode()
1627 	adev->mes.uc_start_addr[pipe] =  in amdgpu_mes_init_microcode()
1630 	adev->mes.data_start_addr[pipe] =  in amdgpu_mes_init_microcode()
1633 	ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data +  in amdgpu_mes_init_microcode()
1635 	adev->mes.fw_version[pipe] =  in amdgpu_mes_init_microcode()
1651 		info->fw = adev->mes.fw[pipe];  in amdgpu_mes_init_microcode()
1658 		info->fw = adev->mes.fw[pipe];  in amdgpu_mes_init_microcode()
1666 	amdgpu_ucode_release(&adev->mes.fw[pipe]);  in amdgpu_mes_init_microcode()
1672 	uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;  in amdgpu_mes_suspend_resume_all_supported()
1693 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_set_enforce_isolation()
1699 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_set_enforce_isolation()
1729 	uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr);  in amdgpu_debugfs_mes_event_log_show()
1732 		     mem, adev->mes.event_log_size, false);  in amdgpu_debugfs_mes_event_log_show()