Lines Matching defs:adev
236 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev);
237 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev);
238 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev);
239 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev);
240 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev);
241 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev);
242 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
244 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev);
245 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
247 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
252 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
256 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
257 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
258 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
323 struct amdgpu_device *adev = kiq_ring->adev;
326 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
327 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
392 static void gfx_v12_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
394 adev->gfx.kiq[0].pmf = &gfx_v12_0_kiq_pm4_funcs;
421 struct amdgpu_device *adev = ring->adev;
430 dev_err(adev->dev,
446 for (i = 0; i < adev->usec_timeout; i++) {
456 if (i >= adev->usec_timeout)
463 struct amdgpu_device *adev = ring->adev;
472 if (adev->enable_mes_kiq &&
492 r = amdgpu_device_wb_get(adev, &index);
496 gpu_addr = adev->wb.gpu_addr + (index * 4);
497 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
498 cpu_ptr = &adev->wb.wb[index];
500 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
502 dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
536 amdgpu_device_wb_free(adev, index);
540 static void gfx_v12_0_free_microcode(struct amdgpu_device *adev)
542 amdgpu_ucode_release(&adev->gfx.pfp_fw);
543 amdgpu_ucode_release(&adev->gfx.me_fw);
544 amdgpu_ucode_release(&adev->gfx.rlc_fw);
545 amdgpu_ucode_release(&adev->gfx.mec_fw);
547 kfree(adev->gfx.rlc.register_list_format);
550 static int gfx_v12_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
555 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
561 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
562 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
563 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
564 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
565 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
569 amdgpu_ucode_release(&adev->psp.toc_fw);
573 static int gfx_v12_0_init_microcode(struct amdgpu_device *adev)
583 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
585 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
590 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
591 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
593 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
598 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
599 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
601 if (!amdgpu_sriov_vf(adev)) {
602 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
607 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
610 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
615 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
620 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
621 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
622 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
624 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
625 err = gfx_v12_0_init_toc_microcode(adev, ucode_prefix);
628 adev->gfx.mec2_fw = NULL;
630 if (adev->gfx.imu.funcs) {
631 if (adev->gfx.imu.funcs->init_microcode) {
632 err = adev->gfx.imu.funcs->init_microcode(adev);
634 dev_err(adev->dev, "Failed to load imu firmware!\n");
640 amdgpu_ucode_release(&adev->gfx.pfp_fw);
641 amdgpu_ucode_release(&adev->gfx.me_fw);
642 amdgpu_ucode_release(&adev->gfx.rlc_fw);
643 amdgpu_ucode_release(&adev->gfx.mec_fw);
649 static u32 gfx_v12_0_get_csb_size(struct amdgpu_device *adev)
668 static void gfx_v12_0_get_csb_buffer(struct amdgpu_device *adev,
675 if (adev->gfx.rlc.cs_data == NULL)
682 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
699 static void gfx_v12_0_rlc_fini(struct amdgpu_device *adev)
702 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
703 &adev->gfx.rlc.clear_state_gpu_addr,
704 (void **)&adev->gfx.rlc.cs_ptr);
707 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
708 &adev->gfx.rlc.cp_table_gpu_addr,
709 (void **)&adev->gfx.rlc.cp_table_ptr);
712 static void gfx_v12_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
716 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
724 adev->gfx.rlc.rlcg_reg_access_supported = true;
727 static int gfx_v12_0_rlc_init(struct amdgpu_device *adev)
732 adev->gfx.rlc.cs_data = gfx12_cs_data;
734 cs_data = adev->gfx.rlc.cs_data;
738 r = amdgpu_gfx_rlc_init_csb(adev);
744 if (adev->gfx.rlc.funcs->update_spm_vmid)
745 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
750 static void gfx_v12_0_mec_fini(struct amdgpu_device *adev)
752 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
753 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
754 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
757 static void gfx_v12_0_me_init(struct amdgpu_device *adev)
759 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
761 amdgpu_gfx_graphics_queue_acquire(adev);
764 static int gfx_v12_0_mec_init(struct amdgpu_device *adev)
770 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
773 amdgpu_gfx_compute_queue_acquire(adev);
774 mec_hpd_size = adev->gfx.num_compute_rings * GFX12_MEC_HPD_SIZE;
777 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
779 &adev->gfx.mec.hpd_eop_obj,
780 &adev->gfx.mec.hpd_eop_gpu_addr,
783 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
784 gfx_v12_0_mec_fini(adev);
790 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
791 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
797 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
805 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
818 static void gfx_v12_0_read_wave_data(struct amdgpu_device *adev,
830 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
831 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
832 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
833 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
834 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
835 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
836 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
837 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
838 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
839 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
840 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
841 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
842 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
843 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
844 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATE_PRIV);
845 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
846 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXCP_FLAG_USER);
847 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAP_CTRL);
848 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_ACTIVE);
849 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_VALID_AND_IDLE);
850 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
851 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
852 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_SCHED_MODE);
855 static void gfx_v12_0_read_wave_sgprs(struct amdgpu_device *adev,
863 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
867 static void gfx_v12_0_read_wave_vgprs(struct amdgpu_device *adev,
874 adev, wave, thread,
878 static void gfx_v12_0_select_me_pipe_q(struct amdgpu_device *adev,
881 soc24_grbm_select(adev, me, pipe, q, vm);
894 static int gfx_v12_0_gpu_early_init(struct amdgpu_device *adev)
897 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
900 adev->gfx.config.max_hw_contexts = 8;
901 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
902 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
903 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
904 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
914 static int gfx_v12_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
921 ring = &adev->gfx.gfx_ring[ring_id];
931 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
933 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
938 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
945 static int gfx_v12_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
953 ring = &adev->gfx.compute_ring[ring_id];
962 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
963 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
969 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
971 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
974 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
994 static void gfx_v12_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1009 static uint32_t gfx_v12_0_calc_toc_total_size(struct amdgpu_device *adev)
1014 gfx_v12_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1029 static int gfx_v12_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1034 total_size = gfx_v12_0_calc_toc_total_size(adev);
1036 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1038 &adev->gfx.rlc.rlc_autoload_bo,
1039 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1040 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1043 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1050 static void gfx_v12_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1057 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1078 gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
1084 data = adev->psp.toc.start_addr;
1090 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
1095 gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
1107 adev->gfx.pfp_fw->data;
1109 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1112 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP,
1115 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1118 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P0_STACK,
1120 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_PFP_P1_STACK,
1124 adev->gfx.me_fw->data;
1126 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1129 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME,
1132 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1135 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P0_STACK,
1137 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_ME_P1_STACK,
1141 adev->gfx.mec_fw->data;
1143 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1146 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
1149 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1152 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
1154 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
1156 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
1158 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
1163 adev->gfx.rlc_fw->data;
1164 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1167 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
1174 rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1176 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1179 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
1182 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1185 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
1189 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1191 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1194 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
1197 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1200 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
1207 gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
1214 adev->sdma.instance[0].fw->data;
1215 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1219 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
1224 gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
1241 adev->mes.fw[pipe]->data;
1243 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1247 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
1249 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1253 gfx_v12_0_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
1257 static int gfx_v12_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1264 gfx_v12_0_rlc_backdoor_autoload_copy_sdma_ucode(adev);
1265 gfx_v12_0_rlc_backdoor_autoload_copy_gfx_ucode(adev);
1266 gfx_v12_0_rlc_backdoor_autoload_copy_mes_ucode(adev);
1267 gfx_v12_0_rlc_backdoor_autoload_copy_toc_ucode(adev);
1271 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
1278 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
1280 if (adev->gfx.imu.funcs->load_microcode)
1281 adev->gfx.imu.funcs->load_microcode(adev);
1283 if (adev->gfx.imu.funcs->setup_imu)
1284 adev->gfx.imu.funcs->setup_imu(adev);
1285 if (adev->gfx.imu.funcs->start_imu)
1286 adev->gfx.imu.funcs->start_imu(adev);
1289 gfx_v12_0_disable_gpa_mode(adev);
1302 static void gfx_v12_0_alloc_ip_dump(struct amdgpu_device *adev)
1311 adev->gfx.ip_dump_core = NULL;
1313 adev->gfx.ip_dump_core = ptr;
1318 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1319 adev->gfx.mec.num_queue_per_pipe;
1324 adev->gfx.ip_dump_compute_queues = NULL;
1326 adev->gfx.ip_dump_compute_queues = ptr;
1331 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1332 adev->gfx.me.num_queue_per_pipe;
1337 adev->gfx.ip_dump_gfx_queues = NULL;
1339 adev->gfx.ip_dump_gfx_queues = ptr;
1348 struct amdgpu_device *adev = ip_block->adev;
1350 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler);
1352 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1355 adev->gfx.me.num_me = 1;
1356 adev->gfx.me.num_pipe_per_me = 1;
1357 adev->gfx.me.num_queue_per_pipe = 1;
1358 adev->gfx.mec.num_mec = 1;
1359 adev->gfx.mec.num_pipe_per_mec = 2;
1360 adev->gfx.mec.num_queue_per_pipe = 4;
1363 adev->gfx.me.num_me = 1;
1364 adev->gfx.me.num_pipe_per_me = 1;
1365 adev->gfx.me.num_queue_per_pipe = 1;
1366 adev->gfx.mec.num_mec = 1;
1367 adev->gfx.mec.num_pipe_per_mec = 4;
1368 adev->gfx.mec.num_queue_per_pipe = 8;
1372 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1375 if (adev->gfx.me_fw_version >= 2480 &&
1376 adev->gfx.pfp_fw_version >= 2530 &&
1377 adev->gfx.mec_fw_version >= 2680 &&
1378 adev->mes.fw_version[0] >= 100)
1379 adev->gfx.enable_cleaner_shader = true;
1382 adev->gfx.enable_cleaner_shader = false;
1387 num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
1388 adev->gfx.mec.num_queue_per_pipe) / 2;
1389 adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
1393 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1395 &adev->gfx.eop_irq);
1400 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1402 &adev->gfx.bad_op_irq);
1407 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1409 &adev->gfx.priv_reg_irq);
1414 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1416 &adev->gfx.priv_inst_irq);
1420 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1422 gfx_v12_0_me_init(adev);
1424 r = gfx_v12_0_rlc_init(adev);
1426 dev_err(adev->dev, "Failed to init rlc BOs!\n");
1430 r = gfx_v12_0_mec_init(adev);
1432 dev_err(adev->dev, "Failed to init MEC BOs!\n");
1437 for (i = 0; i < adev->gfx.me.num_me; i++) {
1438 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1439 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1440 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1443 r = gfx_v12_0_gfx_ring_init(adev, ring_id,
1454 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1455 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1456 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1457 if (!amdgpu_gfx_is_mec_queue_enabled(adev,
1461 r = gfx_v12_0_compute_ring_init(adev, ring_id,
1471 adev->gfx.gfx_supported_reset =
1472 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
1473 adev->gfx.compute_supported_reset =
1474 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
1475 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1478 if ((adev->gfx.me_fw_version >= 2660) &&
1479 (adev->gfx.mec_fw_version >= 2920)) {
1480 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1481 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1485 if (!adev->enable_mes_kiq) {
1486 r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, 0);
1488 dev_err(adev->dev, "Failed to init KIQ BOs!\n");
1492 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1497 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_compute_mqd), 0);
1502 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1503 r = gfx_v12_0_rlc_autoload_buffer_init(adev);
1508 r = gfx_v12_0_gpu_early_init(adev);
1512 gfx_v12_0_alloc_ip_dump(adev);
1514 r = amdgpu_gfx_sysfs_init(adev);
1521 static void gfx_v12_0_pfp_fini(struct amdgpu_device *adev)
1523 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1524 &adev->gfx.pfp.pfp_fw_gpu_addr,
1525 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1527 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1528 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1529 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1532 static void gfx_v12_0_me_fini(struct amdgpu_device *adev)
1534 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1535 &adev->gfx.me.me_fw_gpu_addr,
1536 (void **)&adev->gfx.me.me_fw_ptr);
1538 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1539 &adev->gfx.me.me_fw_data_gpu_addr,
1540 (void **)&adev->gfx.me.me_fw_data_ptr);
1543 static void gfx_v12_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1545 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1546 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1547 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1553 struct amdgpu_device *adev = ip_block->adev;
1555 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1556 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1557 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1558 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1560 amdgpu_gfx_mqd_sw_fini(adev, 0);
1562 if (!adev->enable_mes_kiq) {
1563 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1564 amdgpu_gfx_kiq_fini(adev, 0);
1567 gfx_v12_0_pfp_fini(adev);
1568 gfx_v12_0_me_fini(adev);
1569 gfx_v12_0_rlc_fini(adev);
1570 gfx_v12_0_mec_fini(adev);
1572 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1573 gfx_v12_0_rlc_autoload_buffer_fini(adev);
1575 gfx_v12_0_free_microcode(adev);
1577 amdgpu_gfx_sysfs_fini(adev);
1579 kfree(adev->gfx.ip_dump_core);
1580 kfree(adev->gfx.ip_dump_compute_queues);
1581 kfree(adev->gfx.ip_dump_gfx_queues);
1586 static void gfx_v12_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1613 static u32 gfx_v12_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1625 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1626 adev->gfx.config.max_shader_engines);
1631 static u32 gfx_v12_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1644 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1645 adev->gfx.config.max_shader_engines);
1650 static void gfx_v12_0_setup_rb(struct amdgpu_device *adev)
1661 active_sa_bitmap = gfx_v12_0_get_sa_active_bitmap(adev);
1663 global_active_rb_bitmap = gfx_v12_0_get_rb_active_bitmap(adev);
1666 max_sa = adev->gfx.config.max_shader_engines *
1667 adev->gfx.config.max_sh_per_se;
1668 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1669 adev->gfx.config.max_sh_per_se;
1678 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1679 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1685 static void gfx_v12_0_init_compute_vmid(struct amdgpu_device *adev)
1700 mutex_lock(&adev->srbm_mutex);
1701 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1702 soc24_grbm_select(adev, 0, 0, 0, i);
1712 soc24_grbm_select(adev, 0, 0, 0, 0);
1713 mutex_unlock(&adev->srbm_mutex);
1716 static void gfx_v12_0_tcp_harvest(struct amdgpu_device *adev)
1721 static void gfx_v12_0_get_tcc_info(struct amdgpu_device *adev)
1725 static void gfx_v12_0_constants_init(struct amdgpu_device *adev)
1730 if (!amdgpu_sriov_vf(adev))
1733 gfx_v12_0_setup_rb(adev);
1734 gfx_v12_0_get_cu_info(adev, &adev->gfx.cu_info);
1735 gfx_v12_0_get_tcc_info(adev);
1736 adev->gfx.config.pa_sc_tile_steering_override = 0;
1740 mutex_lock(&adev->srbm_mutex);
1741 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1742 soc24_grbm_select(adev, 0, 0, 0, i);
1747 (adev->gmc.private_aperture_start >> 48));
1749 (adev->gmc.shared_aperture_start >> 48));
1753 soc24_grbm_select(adev, 0, 0, 0, 0);
1755 mutex_unlock(&adev->srbm_mutex);
1757 gfx_v12_0_init_compute_vmid(adev);
1760 static u32 gfx_v12_0_get_cpg_int_cntl(struct amdgpu_device *adev,
1774 static u32 gfx_v12_0_get_cpc_int_cntl(struct amdgpu_device *adev,
1795 static void gfx_v12_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1801 if (amdgpu_sriov_vf(adev))
1804 for (i = 0; i < adev->gfx.me.num_me; i++) {
1805 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
1806 cp_int_cntl_reg = gfx_v12_0_get_cpg_int_cntl(adev, i, j);
1824 static int gfx_v12_0_init_csb(struct amdgpu_device *adev)
1826 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1829 adev->gfx.rlc.clear_state_gpu_addr >> 32);
1831 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1832 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1837 static void gfx_v12_0_rlc_stop(struct amdgpu_device *adev)
1845 static void gfx_v12_0_rlc_reset(struct amdgpu_device *adev)
1853 static void gfx_v12_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1875 static void gfx_v12_0_rlc_start(struct amdgpu_device *adev)
1880 gfx_v12_0_rlc_smu_handshake_cntl(adev, false);
1886 static void gfx_v12_0_rlc_enable_srm(struct amdgpu_device *adev)
1897 static void gfx_v12_0_load_rlcg_microcode(struct amdgpu_device *adev)
1903 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1904 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1915 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1918 static void gfx_v12_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
1925 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1927 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1940 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1942 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1954 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1962 static int gfx_v12_0_rlc_load_microcode(struct amdgpu_device *adev)
1968 if (!adev->gfx.rlc_fw)
1971 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1978 gfx_v12_0_load_rlcg_microcode(adev);
1981 gfx_v12_0_load_rlc_iram_dram_microcode(adev);
1990 static int gfx_v12_0_rlc_resume(struct amdgpu_device *adev)
1994 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1995 gfx_v12_0_init_csb(adev);
1997 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
1998 gfx_v12_0_rlc_enable_srm(adev);
2000 if (amdgpu_sriov_vf(adev)) {
2001 gfx_v12_0_init_csb(adev);
2005 adev->gfx.rlc.funcs->stop(adev);
2013 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2015 r = gfx_v12_0_rlc_load_microcode(adev);
2020 gfx_v12_0_init_csb(adev);
2022 adev->gfx.rlc.funcs->start(adev);
2028 static void gfx_v12_0_config_gfx_rs64(struct amdgpu_device *adev)
2036 adev->gfx.mec_fw->data;
2038 adev->gfx.me_fw->data;
2040 adev->gfx.pfp_fw->data;
2044 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2051 soc24_grbm_select(adev, 0, 0, 0, 0);
2066 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2073 soc24_grbm_select(adev, 0, 0, 0, 0);
2088 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
2095 soc24_grbm_select(adev, 0, 0, 0, 0);
2113 static void gfx_v12_0_set_pfp_ucode_start_addr(struct amdgpu_device *adev)
2119 adev->gfx.pfp_fw->data;
2120 mutex_lock(&adev->srbm_mutex);
2121 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2122 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2151 soc24_grbm_select(adev, 0, 0, 0, 0);
2152 mutex_unlock(&adev->srbm_mutex);
2155 static void gfx_v12_0_set_me_ucode_start_addr(struct amdgpu_device *adev)
2161 adev->gfx.me_fw->data;
2162 mutex_lock(&adev->srbm_mutex);
2163 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2164 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2193 soc24_grbm_select(adev, 0, 0, 0, 0);
2194 mutex_unlock(&adev->srbm_mutex);
2197 static void gfx_v12_0_set_mec_ucode_start_addr(struct amdgpu_device *adev)
2203 adev->gfx.mec_fw->data;
2204 mutex_lock(&adev->srbm_mutex);
2205 for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
2206 soc24_grbm_select(adev, 1, pipe_id, 0, 0);
2213 soc24_grbm_select(adev, 0, 0, 0, 0);
2214 mutex_unlock(&adev->srbm_mutex);
2217 static int gfx_v12_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2223 for (i = 0; i < adev->usec_timeout; i++) {
2237 if (i >= adev->usec_timeout) {
2238 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2242 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2243 gfx_v12_0_set_pfp_ucode_start_addr(adev);
2244 gfx_v12_0_set_me_ucode_start_addr(adev);
2245 gfx_v12_0_set_mec_ucode_start_addr(adev);
2251 static int gfx_v12_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2260 for (i = 0; i < adev->usec_timeout; i++) {
2266 if (i >= adev->usec_timeout)
2272 static int gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2282 adev->gfx.pfp_fw->data;
2287 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
2291 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2296 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2298 &adev->gfx.pfp.pfp_fw_obj,
2299 &adev->gfx.pfp.pfp_fw_gpu_addr,
2300 (void **)&adev->gfx.pfp.pfp_fw_ptr);
2302 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
2303 gfx_v12_0_pfp_fini(adev);
2307 r = amdgpu_bo_create_reserved(adev, fw_data_size,
2309 &adev->gfx.pfp.pfp_fw_data_obj,
2310 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
2311 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
2313 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
2314 gfx_v12_0_pfp_fini(adev);
2318 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
2319 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
2321 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2322 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
2323 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2324 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
2327 adev->hdp.funcs->flush_hdp(adev, NULL);
2330 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2332 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2354 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2372 dev_err(adev->dev, "failed to prime instruction cache\n");
2376 mutex_lock(&adev->srbm_mutex);
2377 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2378 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2381 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2383 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
2385 soc24_grbm_select(adev, 0, 0, 0, 0);
2386 mutex_unlock(&adev->srbm_mutex);
2407 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2411 gfx_v12_0_set_pfp_ucode_start_addr(adev);
2416 static int gfx_v12_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
2426 adev->gfx.me_fw->data;
2431 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
2435 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2440 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2442 &adev->gfx.me.me_fw_obj,
2443 &adev->gfx.me.me_fw_gpu_addr,
2444 (void **)&adev->gfx.me.me_fw_ptr);
2446 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
2447 gfx_v12_0_me_fini(adev);
2451 r = amdgpu_bo_create_reserved(adev, fw_data_size,
2453 &adev->gfx.me.me_fw_data_obj,
2454 &adev->gfx.me.me_fw_data_gpu_addr,
2455 (void **)&adev->gfx.me.me_fw_data_ptr);
2457 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
2458 gfx_v12_0_me_fini(adev);
2462 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
2463 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
2465 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2466 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
2467 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2468 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
2471 adev->hdp.funcs->flush_hdp(adev, NULL);
2474 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
2476 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
2498 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2517 dev_err(adev->dev, "failed to prime instruction cache\n");
2521 mutex_lock(&adev->srbm_mutex);
2522 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2523 soc24_grbm_select(adev, 0, pipe_id, 0, 0);
2526 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2528 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
2530 soc24_grbm_select(adev, 0, 0, 0, 0);
2531 mutex_unlock(&adev->srbm_mutex);
2552 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2556 gfx_v12_0_set_me_ucode_start_addr(adev);
2561 static int gfx_v12_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2565 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
2568 gfx_v12_0_cp_gfx_enable(adev, false);
2570 r = gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(adev);
2572 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
2576 r = gfx_v12_0_cp_gfx_load_me_microcode_rs64(adev);
2578 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
2585 static int gfx_v12_0_cp_gfx_start(struct amdgpu_device *adev)
2589 adev->gfx.config.max_hw_contexts - 1);
2593 gfx_v12_0_cp_gfx_enable(adev, true);
2598 static void gfx_v12_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
2609 static void gfx_v12_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
2634 static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev)
2648 mutex_lock(&adev->srbm_mutex);
2649 gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2652 ring = &adev->gfx.gfx_ring[0];
2684 gfx_v12_0_cp_gfx_set_doorbell(adev, ring);
2685 mutex_unlock(&adev->srbm_mutex);
2688 mutex_lock(&adev->srbm_mutex);
2689 gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2690 mutex_unlock(&adev->srbm_mutex);
2693 gfx_v12_0_cp_gfx_start(adev);
2697 static void gfx_v12_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2724 adev->gfx.kiq[0].ring.sched.ready = enable;
2729 static int gfx_v12_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
2738 if (!adev->gfx.mec_fw)
2741 gfx_v12_0_cp_compute_enable(adev, false);
2743 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
2746 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
2750 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
2754 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2756 &adev->gfx.mec.mec_fw_obj,
2757 &adev->gfx.mec.mec_fw_gpu_addr,
2760 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2761 gfx_v12_0_mec_fini(adev);
2765 r = amdgpu_bo_create_reserved(adev,
2767 adev->gfx.mec.num_pipe_per_mec,
2769 &adev->gfx.mec.mec_fw_data_obj,
2770 &adev->gfx.mec.mec_fw_data_gpu_addr,
2773 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2774 gfx_v12_0_mec_fini(adev);
2779 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2783 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2784 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
2785 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2786 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
2799 mutex_lock(&adev->srbm_mutex);
2800 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2801 soc24_grbm_select(adev, 1, i, 0, 0);
2804 lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2807 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2811 lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2813 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2815 mutex_unlock(&adev->srbm_mutex);
2816 soc24_grbm_select(adev, 0, 0, 0, 0);
2833 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2852 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2856 gfx_v12_0_set_mec_ucode_start_addr(adev);
2864 struct amdgpu_device *adev = ring->adev;
2873 static void gfx_v12_0_cp_set_doorbell_range(struct amdgpu_device *adev)
2877 (adev->doorbell_index.gfx_ring0 * 2) << 2);
2879 (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
2883 (adev->doorbell_index.kiq * 2) << 2);
2885 (adev->doorbell_index.userqueue_end * 2) << 2);
2888 static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
2976 struct amdgpu_device *adev = ring->adev;
2978 int mqd_idx = ring - &adev->gfx.gfx_ring[0];
2980 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
2982 mutex_lock(&adev->srbm_mutex);
2983 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
2985 soc24_grbm_select(adev, 0, 0, 0, 0);
2986 mutex_unlock(&adev->srbm_mutex);
2987 if (adev->gfx.me.mqd_backup[mqd_idx])
2988 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
2991 if (adev->gfx.me.mqd_backup[mqd_idx])
2992 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
3002 static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3006 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3007 r = gfx_v12_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false);
3012 r = amdgpu_gfx_enable_kgq(adev, 0);
3016 return gfx_v12_0_cp_gfx_start(adev);
3019 static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
3150 struct amdgpu_device *adev = ring->adev;
3155 if (amdgpu_sriov_vf(adev))
3178 for (j = 0; j < adev->usec_timeout; j++) {
3228 (adev->doorbell_index.kiq * 2) << 2);
3230 (adev->doorbell_index.userqueue_end * 2) << 2);
3260 struct amdgpu_device *adev = ring->adev;
3266 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3268 if (adev->gfx.mec.mqd_backup[mqd_idx])
3269 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3275 mutex_lock(&adev->srbm_mutex);
3276 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3278 soc24_grbm_select(adev, 0, 0, 0, 0);
3279 mutex_unlock(&adev->srbm_mutex);
3282 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3284 mutex_lock(&adev->srbm_mutex);
3285 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3288 soc24_grbm_select(adev, 0, 0, 0, 0);
3289 mutex_unlock(&adev->srbm_mutex);
3291 if (adev->gfx.mec.mqd_backup[mqd_idx])
3292 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3300 struct amdgpu_device *adev = ring->adev;
3302 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3304 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
3306 mutex_lock(&adev->srbm_mutex);
3307 soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3309 soc24_grbm_select(adev, 0, 0, 0, 0);
3310 mutex_unlock(&adev->srbm_mutex);
3312 if (adev->gfx.mec.mqd_backup[mqd_idx])
3313 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3316 if (adev->gfx.mec.mqd_backup[mqd_idx])
3317 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3327 static int gfx_v12_0_kiq_resume(struct amdgpu_device *adev)
3329 gfx_v12_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
3330 adev->gfx.kiq[0].ring.sched.ready = true;
3334 static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev)
3339 gfx_v12_0_cp_compute_enable(adev, true);
3341 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3342 r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
3347 return amdgpu_gfx_enable_kcq(adev, 0);
3350 static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
3355 if (!(adev->flags & AMD_IS_APU))
3356 gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3358 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3360 r = gfx_v12_0_cp_gfx_load_microcode(adev);
3364 r = gfx_v12_0_cp_compute_load_microcode_rs64(adev);
3369 gfx_v12_0_cp_set_doorbell_range(adev);
3372 gfx_v12_0_cp_compute_enable(adev, true);
3373 gfx_v12_0_cp_gfx_enable(adev, true);
3376 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
3377 r = amdgpu_mes_kiq_hw_init(adev);
3379 r = gfx_v12_0_kiq_resume(adev);
3383 r = gfx_v12_0_kcq_resume(adev);
3388 r = gfx_v12_0_cp_gfx_resume(adev);
3392 r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
3397 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3398 ring = &adev->gfx.gfx_ring[i];
3404 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3405 ring = &adev->gfx.compute_ring[i];
3414 static void gfx_v12_0_cp_enable(struct amdgpu_device *adev, bool enable)
3416 gfx_v12_0_cp_gfx_enable(adev, enable);
3417 gfx_v12_0_cp_compute_enable(adev, enable);
3420 static int gfx_v12_0_gfxhub_enable(struct amdgpu_device *adev)
3425 r = adev->gfxhub.funcs->gart_enable(adev);
3429 adev->hdp.funcs->flush_hdp(adev, NULL);
3434 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
3437 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
3442 static int get_gb_addr_config(struct amdgpu_device *adev)
3450 adev->gfx.config.gb_addr_config_fields.num_pkrs =
3453 adev->gfx.config.gb_addr_config = gb_addr_config;
3455 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
3456 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3459 adev->gfx.config.max_tile_pipes =
3460 adev->gfx.config.gb_addr_config_fields.num_pipes;
3462 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
3463 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3465 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
3466 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3468 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
3469 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3471 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
3472 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
3478 static void gfx_v12_0_disable_gpa_mode(struct amdgpu_device *adev)
3491 static void gfx_v12_0_init_golden_registers(struct amdgpu_device *adev)
3493 if (amdgpu_sriov_vf(adev))
3496 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3499 soc15_program_register_sequence(adev,
3503 if (adev->rev_id == 0)
3504 soc15_program_register_sequence(adev,
3516 struct amdgpu_device *adev = ip_block->adev;
3518 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3519 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3521 if (adev->gfx.imu.funcs->program_rlc_ram)
3522 adev->gfx.imu.funcs->program_rlc_ram(adev);
3525 r = gfx_v12_0_rlc_backdoor_autoload_enable(adev);
3529 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3530 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
3531 if (adev->gfx.imu.funcs->load_microcode)
3532 adev->gfx.imu.funcs->load_microcode(adev);
3533 if (adev->gfx.imu.funcs->setup_imu)
3534 adev->gfx.imu.funcs->setup_imu(adev);
3535 if (adev->gfx.imu.funcs->start_imu)
3536 adev->gfx.imu.funcs->start_imu(adev);
3540 gfx_v12_0_disable_gpa_mode(adev);
3544 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
3545 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
3546 r = gfx_v12_0_wait_for_rlc_autoload_complete(adev);
3548 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
3554 gfx_v12_0_init_golden_registers(adev);
3556 adev->gfx.is_poweron = true;
3558 if (get_gb_addr_config(adev))
3561 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
3562 gfx_v12_0_config_gfx_rs64(adev);
3564 r = gfx_v12_0_gfxhub_enable(adev);
3568 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT ||
3569 adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) &&
3576 r = amdgpu_pm_load_smu_firmware(adev, NULL);
3581 gfx_v12_0_constants_init(adev);
3583 if (adev->nbio.funcs->gc_doorbell_init)
3584 adev->nbio.funcs->gc_doorbell_init(adev);
3586 r = gfx_v12_0_rlc_resume(adev);
3594 gfx_v12_0_tcp_harvest(adev);
3596 r = gfx_v12_0_cp_resume(adev);
3605 struct amdgpu_device *adev = ip_block->adev;
3608 cancel_delayed_work_sync(&adev->gfx.idle_work);
3610 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3611 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3612 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
3614 if (!adev->no_hw_access) {
3616 if (amdgpu_gfx_disable_kgq(adev, 0))
3620 if (amdgpu_gfx_disable_kcq(adev, 0))
3623 amdgpu_mes_kiq_hw_fini(adev);
3626 if (amdgpu_sriov_vf(adev)) {
3627 gfx_v12_0_cp_gfx_enable(adev, false);
3635 gfx_v12_0_cp_enable(adev, false);
3636 gfx_v12_0_enable_gui_idle_interrupt(adev, false);
3638 adev->gfxhub.funcs->gart_disable(adev);
3640 adev->gfx.is_poweron = false;
3657 struct amdgpu_device *adev = ip_block->adev;
3670 struct amdgpu_device *adev = ip_block->adev;
3672 for (i = 0; i < adev->usec_timeout; i++) {
3684 static uint64_t gfx_v12_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3688 if (adev->smuio.funcs &&
3689 adev->smuio.funcs->get_gpu_clock_counter)
3690 clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
3692 dev_warn(adev->dev, "query gpu clock counter is not supported\n");
3699 struct amdgpu_device *adev = ip_block->adev;
3701 adev->gfx.funcs = &gfx_v12_0_gfx_funcs;
3703 adev->gfx.num_gfx_rings = GFX12_NUM_GFX_RINGS;
3704 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
3707 gfx_v12_0_set_kiq_pm4_funcs(adev);
3708 gfx_v12_0_set_ring_funcs(adev);
3709 gfx_v12_0_set_irq_funcs(adev);
3710 gfx_v12_0_set_rlc_funcs(adev);
3711 gfx_v12_0_set_mqd_funcs(adev);
3712 gfx_v12_0_set_imu_funcs(adev);
3714 gfx_v12_0_init_rlcg_reg_access_ctrl(adev);
3716 return gfx_v12_0_init_microcode(adev);
3721 struct amdgpu_device *adev = ip_block->adev;
3724 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
3728 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
3732 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
3739 static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev)
3748 static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev,
3760 for (i = 0; i < adev->usec_timeout; i++) {
3768 static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev,
3774 static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
3779 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
3793 static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
3800 if (amdgpu_sriov_is_pp_one_vf(adev))
3808 if (amdgpu_sriov_is_pp_one_vf(adev))
3814 && amdgpu_sriov_is_pp_one_vf(adev)
3837 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
3842 static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
3851 struct amdgpu_device *adev = ip_block->adev;
3854 if (amdgpu_sriov_vf(adev))
3857 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3860 amdgpu_gfx_off_ctrl(adev, enable);
3869 static void gfx_v12_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
3874 if (!(adev->cg_flags &
3885 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3887 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3889 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
3890 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3900 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
3906 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
3918 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
3924 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
3955 if (adev->sdma.num_instances > 1) {
3964 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3967 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3976 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
3978 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3986 static void gfx_v12_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
3990 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
3995 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4007 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
4020 static void gfx_v12_0_update_repeater_fgcg(struct amdgpu_device *adev,
4025 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
4041 static void gfx_v12_0_update_sram_fgcg(struct amdgpu_device *adev,
4046 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
4060 static int gfx_v12_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4063 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4065 gfx_v12_0_update_coarse_grain_clock_gating(adev, enable);
4067 gfx_v12_0_update_medium_grain_clock_gating(adev, enable);
4069 gfx_v12_0_update_repeater_fgcg(adev, enable);
4071 gfx_v12_0_update_sram_fgcg(adev, enable);
4073 gfx_v12_0_update_perf_clk(adev, enable);
4075 if (adev->cg_flags &
4081 gfx_v12_0_enable_gui_idle_interrupt(adev, enable);
4083 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4091 struct amdgpu_device *adev = ip_block->adev;
4093 if (amdgpu_sriov_vf(adev))
4096 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4099 gfx_v12_0_update_gfx_clock_gating(adev,
4111 struct amdgpu_device *adev = ip_block->adev;
4158 struct amdgpu_device *adev = ring->adev;
4174 struct amdgpu_device *adev = ring->adev;
4178 uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
4186 amdgpu_mes_get_aggregated_doorbell_index(adev,
4237 struct amdgpu_device *adev = ring->adev;
4241 uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
4249 amdgpu_mes_get_aggregated_doorbell_index(adev,
4279 struct amdgpu_device *adev = ring->adev;
4281 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
4301 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
4302 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
4431 struct amdgpu_device *adev = ring->adev;
4495 struct amdgpu_device *adev = ring->adev;
4496 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4500 if (adev->enable_mes)
4525 for (i = 0; i < adev->usec_timeout; i++) {
4532 if (i >= adev->usec_timeout) {
4555 struct amdgpu_device *adev = ring->adev;
4563 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4565 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4612 struct amdgpu_device *adev = ring->adev;
4619 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4621 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4625 gfx_v12_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4667 static void gfx_v12_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4718 static int gfx_v12_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4725 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
4728 gfx_v12_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
4731 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4734 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4737 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4740 gfx_v12_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4748 static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
4759 if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
4764 spin_lock(&adev->mes.queue_id_lock);
4765 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
4770 spin_unlock(&adev->mes.queue_id_lock);
4779 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4781 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
4785 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4786 ring = &adev->gfx.compute_ring[i];
4803 static int gfx_v12_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4814 for (i = 0; i < adev->gfx.me.num_me; i++) {
4815 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
4816 cp_int_cntl_reg = gfx_v12_0_get_cpg_int_cntl(adev, i, j);
4827 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4828 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4830 cp_int_cntl_reg = gfx_v12_0_get_cpc_int_cntl(adev, i + 1, j);
4849 static int gfx_v12_0_set_bad_op_fault_state(struct amdgpu_device *adev,
4860 for (i = 0; i < adev->gfx.me.num_me; i++) {
4861 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
4862 cp_int_cntl_reg = gfx_v12_0_get_cpg_int_cntl(adev, i, j);
4873 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4874 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4876 cp_int_cntl_reg = gfx_v12_0_get_cpc_int_cntl(adev, i + 1, j);
4894 static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4905 for (i = 0; i < adev->gfx.me.num_me; i++) {
4906 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
4907 cp_int_cntl_reg = gfx_v12_0_get_cpg_int_cntl(adev, i, j);
4926 static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
4939 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4940 ring = &adev->gfx.gfx_ring[i];
4948 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4949 ring = &adev->gfx.compute_ring[i];
4961 static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev,
4966 gfx_v12_0_handle_priv_fault(adev, entry);
4970 static int gfx_v12_0_bad_op_irq(struct amdgpu_device *adev,
4975 gfx_v12_0_handle_priv_fault(adev, entry);
4979 static int gfx_v12_0_priv_inst_irq(struct amdgpu_device *adev,
4984 gfx_v12_0_handle_priv_fault(adev, entry);
5035 struct amdgpu_device *adev = ip_block->adev;
5039 if (!adev->gfx.ip_dump_core)
5045 adev->gfx.ip_dump_core[i]);
5048 if (!adev->gfx.ip_dump_compute_queues)
5053 adev->gfx.mec.num_mec,
5054 adev->gfx.mec.num_pipe_per_mec,
5055 adev->gfx.mec.num_queue_per_pipe);
5057 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
5058 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
5059 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
5064 adev->gfx.ip_dump_compute_queues[index + reg]);
5072 if (!adev->gfx.ip_dump_gfx_queues)
5078 adev->gfx.me.num_me,
5079 adev->gfx.me.num_pipe_per_me,
5080 adev->gfx.me.num_queue_per_pipe);
5082 for (i = 0; i < adev->gfx.me.num_me; i++) {
5083 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
5084 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
5089 adev->gfx.ip_dump_gfx_queues[index + reg]);
5099 struct amdgpu_device *adev = ip_block->adev;
5103 if (!adev->gfx.ip_dump_core)
5106 amdgpu_gfx_off_ctrl(adev, false);
5108 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_12_0[i]));
5109 amdgpu_gfx_off_ctrl(adev, true);
5112 if (!adev->gfx.ip_dump_compute_queues)
5116 amdgpu_gfx_off_ctrl(adev, false);
5117 mutex_lock(&adev->srbm_mutex);
5118 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
5119 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
5120 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
5122 soc24_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
5124 adev->gfx.ip_dump_compute_queues[index + reg] =
5132 soc24_grbm_select(adev, 0, 0, 0, 0);
5133 mutex_unlock(&adev->srbm_mutex);
5134 amdgpu_gfx_off_ctrl(adev, true);
5137 if (!adev->gfx.ip_dump_gfx_queues)
5142 amdgpu_gfx_off_ctrl(adev, false);
5143 mutex_lock(&adev->srbm_mutex);
5144 for (i = 0; i < adev->gfx.me.num_me; i++) {
5145 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
5146 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
5147 soc24_grbm_select(adev, i, j, k, 0);
5150 adev->gfx.ip_dump_gfx_queues[index + reg] =
5158 soc24_grbm_select(adev, 0, 0, 0, 0);
5159 mutex_unlock(&adev->srbm_mutex);
5160 amdgpu_gfx_off_ctrl(adev, true);
5165 struct amdgpu_device *adev = ring->adev;
5168 if (amdgpu_sriov_vf(adev))
5171 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
5173 dev_err(adev->dev, "reset via MES failed %d\n", r);
5179 dev_err(adev->dev, "failed to init kgq\n");
5183 r = amdgpu_mes_map_legacy_queue(adev, ring);
5185 dev_err(adev->dev, "failed to remap kgq\n");
5194 struct amdgpu_device *adev = ring->adev;
5197 if (amdgpu_sriov_vf(adev))
5200 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true);
5202 dev_err(adev->dev, "reset via MMIO failed %d\n", r);
5208 dev_err(adev->dev, "failed to init kcq\n");
5211 r = amdgpu_mes_map_legacy_queue(adev, ring);
5213 dev_err(adev->dev, "failed to remap kcq\n");
5371 static void gfx_v12_0_set_ring_funcs(struct amdgpu_device *adev)
5375 adev->gfx.kiq[0].ring.funcs = &gfx_v12_0_ring_funcs_kiq;
5377 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
5378 adev->gfx.gfx_ring[i].funcs = &gfx_v12_0_ring_funcs_gfx;
5380 for (i = 0; i < adev->gfx.num_compute_rings; i++)
5381 adev->gfx.compute_ring[i].funcs = &gfx_v12_0_ring_funcs_compute;
5404 static void gfx_v12_0_set_irq_funcs(struct amdgpu_device *adev)
5406 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
5407 adev->gfx.eop_irq.funcs = &gfx_v12_0_eop_irq_funcs;
5409 adev->gfx.priv_reg_irq.num_types = 1;
5410 adev->gfx.priv_reg_irq.funcs = &gfx_v12_0_priv_reg_irq_funcs;
5412 adev->gfx.bad_op_irq.num_types = 1;
5413 adev->gfx.bad_op_irq.funcs = &gfx_v12_0_bad_op_irq_funcs;
5415 adev->gfx.priv_inst_irq.num_types = 1;
5416 adev->gfx.priv_inst_irq.funcs = &gfx_v12_0_priv_inst_irq_funcs;
5419 static void gfx_v12_0_set_imu_funcs(struct amdgpu_device *adev)
5421 if (adev->flags & AMD_IS_APU)
5422 adev->gfx.imu.mode = MISSION_MODE;
5424 adev->gfx.imu.mode = DEBUG_MODE;
5426 adev->gfx.imu.funcs = &gfx_v12_0_imu_funcs;
5429 static void gfx_v12_0_set_rlc_funcs(struct amdgpu_device *adev)
5431 adev->gfx.rlc.funcs = &gfx_v12_0_rlc_funcs;
5434 static void gfx_v12_0_set_mqd_funcs(struct amdgpu_device *adev)
5437 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
5439 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
5442 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
5444 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
5448 static void gfx_v12_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
5462 static u32 gfx_v12_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
5472 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
5477 static u32 gfx_v12_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
5482 wgp_active_bitmap = gfx_v12_0_get_wgp_active_bitmap_per_sh(adev);
5495 static int gfx_v12_0_get_cu_info(struct amdgpu_device *adev,
5502 if (!adev || !cu_info)
5507 mutex_lock(&adev->grbm_idx_mutex);
5508 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
5509 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
5510 bitmap = i * adev->gfx.config.max_sh_per_se + j;
5511 if (!((gfx_v12_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
5515 gfx_v12_0_select_se_sh(adev, i, j, 0xffffffff, 0);
5518 adev, disable_masks[i * 2 + j]);
5519 bitmap = gfx_v12_0_get_cu_active_bitmap_per_sh(adev);
5538 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
5547 gfx_v12_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
5548 mutex_unlock(&adev->grbm_idx_mutex);