145 static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,  in mes_v12_0_submit_pkt_and_poll_completion()  argument
151 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_submit_pkt_and_poll_completion()
152 	struct amdgpu_ring *ring = &mes->ring[pipe];  in mes_v12_0_submit_pkt_and_poll_completion()
153 	spinlock_t *ring_lock = &mes->ring_lock[pipe];  in mes_v12_0_submit_pkt_and_poll_completion()
277 static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,  in mes_v12_0_add_hw_queue()  argument
280 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_add_hw_queue()
328 	return mes_v12_0_submit_pkt_and_poll_completion(mes,  in mes_v12_0_add_hw_queue()
334 static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,  in mes_v12_0_remove_hw_queue()  argument
348 	return mes_v12_0_submit_pkt_and_poll_completion(mes,  in mes_v12_0_remove_hw_queue()
386 static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,  in mes_v12_0_reset_queue_mmio()  argument
390 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_reset_queue_mmio()
480 static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,  in mes_v12_0_reset_hw_queue()  argument
496 	if (mes->adev->enable_uni_mes)  in mes_v12_0_reset_hw_queue()
501 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_reset_hw_queue()
506 static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,  in mes_v12_0_map_legacy_queue()  argument
527 	if (mes->adev->enable_uni_mes)  in mes_v12_0_map_legacy_queue()
532 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_map_legacy_queue()
537 static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,  in mes_v12_0_unmap_legacy_queue()  argument
566 	if (mes->adev->enable_uni_mes)  in mes_v12_0_unmap_legacy_queue()
571 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_unmap_legacy_queue()
576 static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes,  in mes_v12_0_suspend_gang()  argument
582 static int mes_v12_0_resume_gang(struct amdgpu_mes *mes,  in mes_v12_0_resume_gang()  argument
588 static int mes_v12_0_query_sched_status(struct amdgpu_mes *mes, int pipe)  in mes_v12_0_query_sched_status()  argument
598 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_query_sched_status()
603 static int mes_v12_0_misc_op(struct amdgpu_mes *mes,  in mes_v12_0_misc_op()  argument
609 	if (mes->adev->enable_uni_mes)  in mes_v12_0_misc_op()
674 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_misc_op()
679 static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)  in mes_v12_0_set_hw_resources_1()  argument
690 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_set_hw_resources_1()
695 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)  in mes_v12_0_set_hw_resources()  argument
698 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_set_hw_resources()
708 		mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;  in mes_v12_0_set_hw_resources()
709 		mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;  in mes_v12_0_set_hw_resources()
715 				mes->compute_hqd_mask[i];  in mes_v12_0_set_hw_resources()
719 				mes->gfx_hqd_mask[i];  in mes_v12_0_set_hw_resources()
723 				mes->sdma_hqd_mask[i];  in mes_v12_0_set_hw_resources()
727 				mes->aggregated_doorbells[i];  in mes_v12_0_set_hw_resources()
731 		mes->sch_ctx_gpu_addr[pipe];  in mes_v12_0_set_hw_resources()
733 		mes->query_status_fence_gpu_addr[pipe];  in mes_v12_0_set_hw_resources()
759 		mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr +  in mes_v12_0_set_hw_resources()
766 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_set_hw_resources()
771 static void mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes *mes)  in mes_v12_0_init_aggregated_doorbell()  argument
773 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_init_aggregated_doorbell()
780 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<  in mes_v12_0_init_aggregated_doorbell()
789 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<  in mes_v12_0_init_aggregated_doorbell()
798 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<  in mes_v12_0_init_aggregated_doorbell()
807 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<  in mes_v12_0_init_aggregated_doorbell()
816 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<  in mes_v12_0_init_aggregated_doorbell()
827 		struct amdgpu_mes *mes, bool enable)  in mes_v12_0_enable_unmapped_doorbell_handling()  argument
829 	struct amdgpu_device *adev = mes->adev;  in mes_v12_0_enable_unmapped_doorbell_handling()
846 static int mes_v12_0_reset_legacy_queue(struct amdgpu_mes *mes,  in mes_v12_0_reset_legacy_queue()  argument
853 		return mes_v12_0_reset_queue_mmio(mes, input->queue_type,  in mes_v12_0_reset_legacy_queue()
879 	if (mes->adev->enable_uni_mes)  in mes_v12_0_reset_legacy_queue()
884 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,  in mes_v12_0_reset_legacy_queue()
910 		adev->mes.fw[pipe]->data;  in mes_v12_0_allocate_ucode_buffer()
912 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v12_0_allocate_ucode_buffer()
919 				      &adev->mes.ucode_fw_obj[pipe],  in mes_v12_0_allocate_ucode_buffer()
920 				      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v12_0_allocate_ucode_buffer()
921 				      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v12_0_allocate_ucode_buffer()
927 	memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);  in mes_v12_0_allocate_ucode_buffer()
929 	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);  in mes_v12_0_allocate_ucode_buffer()
930 	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);  in mes_v12_0_allocate_ucode_buffer()
944 		adev->mes.fw[pipe]->data;  in mes_v12_0_allocate_ucode_data_buffer()
946 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v12_0_allocate_ucode_data_buffer()
953 				      &adev->mes.data_fw_obj[pipe],  in mes_v12_0_allocate_ucode_data_buffer()
954 				      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v12_0_allocate_ucode_data_buffer()
955 				      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v12_0_allocate_ucode_data_buffer()
961 	memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);  in mes_v12_0_allocate_ucode_data_buffer()
963 	amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);  in mes_v12_0_allocate_ucode_data_buffer()
964 	amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);  in mes_v12_0_allocate_ucode_data_buffer()
972 	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],  in mes_v12_0_free_ucode_buffers()
973 			      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v12_0_free_ucode_buffers()
974 			      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v12_0_free_ucode_buffers()
976 	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],  in mes_v12_0_free_ucode_buffers()
977 			      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v12_0_free_ucode_buffers()
978 			      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v12_0_free_ucode_buffers()
993 				if (adev->mes.event_log_size >= (pipe + 1) * log_size) {  in mes_v12_0_enable()
995 						     lower_32_bits(adev->mes.event_log_gpu_addr +  in mes_v12_0_enable()
998 						     upper_32_bits(adev->mes.event_log_gpu_addr +  in mes_v12_0_enable()
1013 			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v12_0_enable()
1063 		ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v12_0_set_ucode_start_addr()
1083 	if (!adev->mes.fw[pipe])  in mes_v12_0_load_microcode()
1104 		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v12_0_load_microcode()
1106 		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v12_0_load_microcode()
1113 		     lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v12_0_load_microcode()
1115 		     upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v12_0_load_microcode()
1147 			      &adev->mes.eop_gpu_obj[pipe],  in mes_v12_0_allocate_eop_buf()
1148 			      &adev->mes.eop_gpu_addr[pipe],  in mes_v12_0_allocate_eop_buf()
1156 	       adev->mes.eop_gpu_obj[pipe]->tbo.base.size);  in mes_v12_0_allocate_eop_buf()
1158 	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);  in mes_v12_0_allocate_eop_buf()
1159 	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);  in mes_v12_0_allocate_eop_buf()
1350 	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);  in mes_v12_0_kiq_enable_queue()
1369 		ring = &adev->mes.ring[pipe];  in mes_v12_0_queue_init()
1398 		adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v12_0_queue_init()
1400 		adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v12_0_queue_init()
1412 	ring = &adev->mes.ring[pipe];  in mes_v12_0_ring_init()
1422 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[pipe];  in mes_v12_0_ring_init()
1451 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];  in mes_v12_0_kiq_ring_init()
1469 		ring = &adev->mes.ring[pipe];  in mes_v12_0_mqd_sw_init()
1485 	adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);  in mes_v12_0_mqd_sw_init()
1486 	if (!adev->mes.mqd_backup[pipe])  in mes_v12_0_mqd_sw_init()
1499 	adev->mes.funcs = &mes_v12_0_funcs;  in mes_v12_0_sw_init()
1500 	adev->mes.kiq_hw_init = &mes_v12_0_kiq_hw_init;  in mes_v12_0_sw_init()
1501 	adev->mes.kiq_hw_fini = &mes_v12_0_kiq_hw_fini;  in mes_v12_0_sw_init()
1502 	adev->mes.enable_legacy_queue_map = true;  in mes_v12_0_sw_init()
1504 	adev->mes.event_log_size = adev->enable_uni_mes ?  in mes_v12_0_sw_init()
1537 		kfree(adev->mes.mqd_backup[pipe]);  in mes_v12_0_sw_fini()
1539 		amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],  in mes_v12_0_sw_fini()
1540 				      &adev->mes.eop_gpu_addr[pipe],  in mes_v12_0_sw_fini()
1542 		amdgpu_ucode_release(&adev->mes.fw[pipe]);  in mes_v12_0_sw_fini()
1545 			amdgpu_bo_free_kernel(&adev->mes.ring[pipe].mqd_obj,  in mes_v12_0_sw_fini()
1546 					      &adev->mes.ring[pipe].mqd_gpu_addr,  in mes_v12_0_sw_fini()
1547 					      &adev->mes.ring[pipe].mqd_ptr);  in mes_v12_0_sw_fini()
1548 			amdgpu_ring_fini(&adev->mes.ring[pipe]);  in mes_v12_0_sw_fini()
1601 	adev->mes.ring[0].sched.ready = false;  in mes_v12_0_kiq_dequeue_sched()
1622 		mes_v12_0_kiq_setting(&adev->mes.ring[AMDGPU_MES_KIQ_PIPE]);  in mes_v12_0_kiq_hw_init()
1658 		r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE);  in mes_v12_0_kiq_hw_init()
1662 		mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_KIQ_PIPE);  in mes_v12_0_kiq_hw_init()
1665 	if (adev->mes.enable_legacy_queue_map) {  in mes_v12_0_kiq_hw_init()
1680 	if (adev->mes.ring[0].sched.ready) {  in mes_v12_0_kiq_hw_fini()
1683 				      &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],  in mes_v12_0_kiq_hw_fini()
1688 		adev->mes.ring[0].sched.ready = false;  in mes_v12_0_kiq_hw_fini()
1701 	if (adev->mes.ring[0].sched.ready)  in mes_v12_0_hw_init()
1725 	mes_v12_0_enable_unmapped_doorbell_handling(&adev->mes, true);  in mes_v12_0_hw_init()
1731 	r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_SCHED_PIPE);  in mes_v12_0_hw_init()
1736 		mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE);  in mes_v12_0_hw_init()
1738 	mes_v12_0_init_aggregated_doorbell(&adev->mes);  in mes_v12_0_hw_init()
1740 	r = mes_v12_0_query_sched_status(&adev->mes, AMDGPU_MES_SCHED_PIPE);  in mes_v12_0_hw_init()
1757 	adev->mes.ring[0].sched.ready = true;  in mes_v12_0_hw_init()