Lines Matching full:gmu
21 static void a6xx_gmu_fault(struct a6xx_gmu *gmu) in a6xx_gmu_fault() argument
23 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_fault()
28 gmu->hung = true; in a6xx_gmu_fault()
39 struct a6xx_gmu *gmu = data; in a6xx_gmu_irq() local
42 status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS); in a6xx_gmu_irq()
43 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status); in a6xx_gmu_irq()
46 dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n"); in a6xx_gmu_irq()
48 a6xx_gmu_fault(gmu); in a6xx_gmu_irq()
52 dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n"); in a6xx_gmu_irq()
55 dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n", in a6xx_gmu_irq()
56 gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS)); in a6xx_gmu_irq()
63 struct a6xx_gmu *gmu = data; in a6xx_hfi_irq() local
66 status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO); in a6xx_hfi_irq()
67 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status); in a6xx_hfi_irq()
70 dev_err_ratelimited(gmu->dev, "GMU firmware fault\n"); in a6xx_hfi_irq()
72 a6xx_gmu_fault(gmu); in a6xx_hfi_irq()
78 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu) in a6xx_gmu_sptprac_is_on() argument
82 /* This can be called from gpu state code so make sure GMU is valid */ in a6xx_gmu_sptprac_is_on()
83 if (!gmu->initialized) in a6xx_gmu_sptprac_is_on()
86 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); in a6xx_gmu_sptprac_is_on()
94 bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu) in a6xx_gmu_gx_is_on() argument
96 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_gx_is_on()
100 /* This can be called from gpu state code so make sure GMU is valid */ in a6xx_gmu_gx_is_on()
101 if (!gmu->initialized) in a6xx_gmu_gx_is_on()
104 /* If GMU is absent, then GX power domain is ON as long as GPU is in active state */ in a6xx_gmu_gx_is_on()
108 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); in a6xx_gmu_gx_is_on()
126 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_set_freq() local
134 if (gpu_freq == gmu->freq) in a6xx_gmu_set_freq()
137 for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++) in a6xx_gmu_set_freq()
138 if (gpu_freq == gmu->gpu_freqs[perf_index]) in a6xx_gmu_set_freq()
142 if (info->bcms && gmu->nr_gpu_bws > 1) { in a6xx_gmu_set_freq()
145 for (bw_index = 0; bw_index < gmu->nr_gpu_bws - 1; bw_index++) { in a6xx_gmu_set_freq()
146 if (bw == gmu->gpu_bw_table[bw_index]) in a6xx_gmu_set_freq()
165 do_div(tmp, gmu->gpu_bw_table[gmu->nr_gpu_bws - 1]); in a6xx_gmu_set_freq()
172 gmu->current_perf_index = perf_index; in a6xx_gmu_set_freq()
173 gmu->freq = gmu->gpu_freqs[perf_index]; in a6xx_gmu_set_freq()
175 trace_msm_gmu_freq_change(gmu->freq, perf_index); in a6xx_gmu_set_freq()
186 if (!gmu->legacy) { in a6xx_gmu_set_freq()
187 a6xx_hfi_set_freq(gmu, perf_index, bw_index); in a6xx_gmu_set_freq()
194 gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0); in a6xx_gmu_set_freq()
196 gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING, in a6xx_gmu_set_freq()
203 gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff); in a6xx_gmu_set_freq()
205 /* Set and clear the OOB for DCVS to trigger the GMU */ in a6xx_gmu_set_freq()
206 a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET); in a6xx_gmu_set_freq()
207 a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET); in a6xx_gmu_set_freq()
209 ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN); in a6xx_gmu_set_freq()
211 dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret); in a6xx_gmu_set_freq()
220 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_get_freq() local
222 return gmu->freq; in a6xx_gmu_get_freq()
225 static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu) in a6xx_gmu_check_idle_level() argument
228 int local = gmu->idle_level; in a6xx_gmu_check_idle_level()
231 if (gmu->idle_level == GMU_IDLE_STATE_SPTP) in a6xx_gmu_check_idle_level()
234 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE); in a6xx_gmu_check_idle_level()
237 if (gmu->idle_level != GMU_IDLE_STATE_IFPC || in a6xx_gmu_check_idle_level()
238 !a6xx_gmu_gx_is_on(gmu)) in a6xx_gmu_check_idle_level()
245 /* Wait for the GMU to get to its most idle state */
246 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu) in a6xx_gmu_wait_for_idle() argument
248 return spin_until(a6xx_gmu_check_idle_level(gmu)); in a6xx_gmu_wait_for_idle()
251 static int a6xx_gmu_start(struct a6xx_gmu *gmu) in a6xx_gmu_start() argument
253 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_start()
258 val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8); in a6xx_gmu_start()
267 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); in a6xx_gmu_start()
273 gmu_write(gmu, REG_A7XX_GMU_GENERAL_9, 0); in a6xx_gmu_start()
275 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0); in a6xx_gmu_start()
278 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0); in a6xx_gmu_start()
280 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val, in a6xx_gmu_start()
284 DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n"); in a6xx_gmu_start()
286 set_bit(GMU_STATUS_FW_START, &gmu->status); in a6xx_gmu_start()
291 static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu) in a6xx_gmu_hfi_start() argument
296 gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1); in a6xx_gmu_hfi_start()
298 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val, in a6xx_gmu_hfi_start()
301 DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n"); in a6xx_gmu_hfi_start()
350 /* Trigger a OOB (out of band) request to the GMU */
351 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) in a6xx_gmu_set_oob() argument
357 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); in a6xx_gmu_set_oob()
362 if (gmu->legacy) { in a6xx_gmu_set_oob()
369 DRM_DEV_ERROR(gmu->dev, in a6xx_gmu_set_oob()
370 "Invalid non-legacy GMU request %s\n", in a6xx_gmu_set_oob()
377 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request); in a6xx_gmu_set_oob()
380 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val, in a6xx_gmu_set_oob()
384 DRM_DEV_ERROR(gmu->dev, in a6xx_gmu_set_oob()
385 "Timeout waiting for GMU OOB set %s: 0x%x\n", in a6xx_gmu_set_oob()
387 gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO)); in a6xx_gmu_set_oob()
390 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack); in a6xx_gmu_set_oob()
395 /* Clear a pending OOB state in the GMU */
396 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) in a6xx_gmu_clear_oob() argument
400 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); in a6xx_gmu_clear_oob()
405 if (gmu->legacy) in a6xx_gmu_clear_oob()
410 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit); in a6xx_gmu_clear_oob()
414 int a6xx_sptprac_enable(struct a6xx_gmu *gmu) in a6xx_sptprac_enable() argument
419 WARN_ON(!gmu->legacy); in a6xx_sptprac_enable()
421 /* Nothing to do if GMU does the power management */ in a6xx_sptprac_enable()
422 if (gmu->idle_level > GMU_IDLE_STATE_ACTIVE) in a6xx_sptprac_enable()
425 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000); in a6xx_sptprac_enable()
427 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, in a6xx_sptprac_enable()
431 DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n", in a6xx_sptprac_enable()
432 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); in a6xx_sptprac_enable()
439 void a6xx_sptprac_disable(struct a6xx_gmu *gmu) in a6xx_sptprac_disable() argument
444 if (!gmu->legacy) in a6xx_sptprac_disable()
448 gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11)); in a6xx_sptprac_disable()
450 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001); in a6xx_sptprac_disable()
452 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, in a6xx_sptprac_disable()
456 DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n", in a6xx_sptprac_disable()
457 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); in a6xx_sptprac_disable()
460 /* Let the GMU know we are starting a boot sequence */
461 static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu) in a6xx_gmu_gfx_rail_on() argument
465 /* Let the GMU know we are getting ready for boot */ in a6xx_gmu_gfx_rail_on()
466 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0); in a6xx_gmu_gfx_rail_on()
469 vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1]; in a6xx_gmu_gfx_rail_on()
471 gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff); in a6xx_gmu_gfx_rail_on()
472 gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff); in a6xx_gmu_gfx_rail_on()
474 /* Let the GMU know the boot sequence has started */ in a6xx_gmu_gfx_rail_on()
475 return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); in a6xx_gmu_gfx_rail_on()
478 static void a6xx_gemnoc_workaround(struct a6xx_gmu *gmu) in a6xx_gemnoc_workaround() argument
480 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gemnoc_workaround()
489 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, BIT(0)); in a6xx_gemnoc_workaround()
492 /* Let the GMU know that we are about to go into slumber */
493 static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) in a6xx_gmu_notify_slumber() argument
497 /* Disable the power counter so the GMU isn't busy */ in a6xx_gmu_notify_slumber()
498 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); in a6xx_gmu_notify_slumber()
501 if (gmu->idle_level < GMU_IDLE_STATE_SPTP) in a6xx_gmu_notify_slumber()
502 a6xx_sptprac_disable(gmu); in a6xx_gmu_notify_slumber()
504 if (!gmu->legacy) { in a6xx_gmu_notify_slumber()
505 ret = a6xx_hfi_send_prep_slumber(gmu); in a6xx_gmu_notify_slumber()
509 /* Tell the GMU to get ready to slumber */ in a6xx_gmu_notify_slumber()
510 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1); in a6xx_gmu_notify_slumber()
512 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); in a6xx_gmu_notify_slumber()
513 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER); in a6xx_gmu_notify_slumber()
516 /* Check to see if the GMU really did slumber */ in a6xx_gmu_notify_slumber()
517 if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE) in a6xx_gmu_notify_slumber()
519 DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n"); in a6xx_gmu_notify_slumber()
525 a6xx_gemnoc_workaround(gmu); in a6xx_gmu_notify_slumber()
528 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); in a6xx_gmu_notify_slumber()
532 static int a6xx_rpmh_start(struct a6xx_gmu *gmu) in a6xx_rpmh_start() argument
537 if (!test_and_clear_bit(GMU_STATUS_PDC_SLEEP, &gmu->status)) in a6xx_rpmh_start()
540 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, BIT(1)); in a6xx_rpmh_start()
542 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val, in a6xx_rpmh_start()
545 DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n"); in a6xx_rpmh_start()
549 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val, in a6xx_rpmh_start()
553 DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n"); in a6xx_rpmh_start()
557 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); in a6xx_rpmh_start()
562 static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) in a6xx_rpmh_stop() argument
567 if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) in a6xx_rpmh_stop()
570 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); in a6xx_rpmh_stop()
572 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, in a6xx_rpmh_stop()
575 DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n"); in a6xx_rpmh_stop()
577 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); in a6xx_rpmh_stop()
579 set_bit(GMU_STATUS_PDC_SLEEP, &gmu->status); in a6xx_rpmh_stop()
590 static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) in a6xx_gmu_rpmh_init() argument
592 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_rpmh_init()
594 struct platform_device *pdev = to_platform_device(gmu->dev); in a6xx_gmu_rpmh_init()
621 gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24)); in a6xx_gmu_rpmh_init()
624 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1); in a6xx_gmu_rpmh_init()
625 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0); in a6xx_gmu_rpmh_init()
626 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0); in a6xx_gmu_rpmh_init()
627 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0); in a6xx_gmu_rpmh_init()
628 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0); in a6xx_gmu_rpmh_init()
629 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, in a6xx_gmu_rpmh_init()
631 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0); in a6xx_gmu_rpmh_init()
632 gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0); in a6xx_gmu_rpmh_init()
633 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520); in a6xx_gmu_rpmh_init()
634 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510); in a6xx_gmu_rpmh_init()
635 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514); in a6xx_gmu_rpmh_init()
644 gmu_write_rscc(gmu, seqmem0_drv0_reg, 0xeaaae5a0); in a6xx_gmu_rpmh_init()
645 gmu_write_rscc(gmu, seqmem0_drv0_reg + 1, 0xe1a1ebab); in a6xx_gmu_rpmh_init()
646 gmu_write_rscc(gmu, seqmem0_drv0_reg + 2, 0xa2e0a581); in a6xx_gmu_rpmh_init()
647 gmu_write_rscc(gmu, seqmem0_drv0_reg + 3, 0xecac82e2); in a6xx_gmu_rpmh_init()
648 gmu_write_rscc(gmu, seqmem0_drv0_reg + 4, 0x0020edad); in a6xx_gmu_rpmh_init()
650 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0); in a6xx_gmu_rpmh_init()
651 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7); in a6xx_gmu_rpmh_init()
652 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1); in a6xx_gmu_rpmh_init()
653 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2); in a6xx_gmu_rpmh_init()
654 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8); in a6xx_gmu_rpmh_init()
723 /* Set up the idle state for the GMU */
724 static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) in a6xx_gmu_power_config() argument
726 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_power_config()
729 /* Disable GMU WB/RB buffer */ in a6xx_gmu_power_config()
730 gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1); in a6xx_gmu_power_config()
731 gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1); in a6xx_gmu_power_config()
732 gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1); in a6xx_gmu_power_config()
738 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400); in a6xx_gmu_power_config()
740 switch (gmu->idle_level) { in a6xx_gmu_power_config()
742 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST, in a6xx_gmu_power_config()
744 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, in a6xx_gmu_power_config()
749 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST, in a6xx_gmu_power_config()
751 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, in a6xx_gmu_power_config()
757 gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0, in a6xx_gmu_power_config()
786 static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu) in a6xx_gmu_fw_load() argument
788 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_fw_load()
801 if (gmu->legacy) { in a6xx_gmu_fw_load()
804 DRM_DEV_ERROR(gmu->dev, in a6xx_gmu_fw_load()
805 "GMU firmware is bigger than the available region\n"); in a6xx_gmu_fw_load()
809 gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START, in a6xx_gmu_fw_load()
823 gmu_write_bulk(gmu, in a6xx_gmu_fw_load()
828 gmu_write_bulk(gmu, in a6xx_gmu_fw_load()
831 } else if (!fw_block_mem(&gmu->icache, blk) && in a6xx_gmu_fw_load()
832 !fw_block_mem(&gmu->dcache, blk) && in a6xx_gmu_fw_load()
833 !fw_block_mem(&gmu->dummy, blk)) { in a6xx_gmu_fw_load()
834 DRM_DEV_ERROR(gmu->dev, in a6xx_gmu_fw_load()
840 ver = gmu_read(gmu, REG_A6XX_GMU_CORE_FW_VERSION); in a6xx_gmu_fw_load()
841 DRM_INFO_ONCE("Loaded GMU firmware v%u.%u.%u\n", in a6xx_gmu_fw_load()
849 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) in a6xx_gmu_fw_start() argument
851 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_fw_start()
860 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1); in a6xx_gmu_fw_start()
861 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1); in a6xx_gmu_fw_start()
868 gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); in a6xx_gmu_fw_start()
870 ret = a6xx_rpmh_start(gmu); in a6xx_gmu_fw_start()
876 "GMU firmware is not loaded\n")) in a6xx_gmu_fw_start()
879 ret = a6xx_gmu_fw_load(gmu); in a6xx_gmu_fw_start()
885 gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0); in a6xx_gmu_fw_start()
886 gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02); in a6xx_gmu_fw_start()
889 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova); in a6xx_gmu_fw_start()
890 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1); in a6xx_gmu_fw_start()
900 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0, in a6xx_gmu_fw_start()
909 gmu_write(gmu, REG_A6XX_GMU_CM3_CFG, 0x4052); in a6xx_gmu_fw_start()
915 * Note that the GMU has a slightly different layout for in a6xx_gmu_fw_start()
927 gmu_write(gmu, REG_A7XX_GMU_GENERAL_10, chipid); in a6xx_gmu_fw_start()
928 gmu_write(gmu, REG_A7XX_GMU_GENERAL_8, in a6xx_gmu_fw_start()
929 (gmu->log.iova & GENMASK(31, 12)) | in a6xx_gmu_fw_start()
930 ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0))); in a6xx_gmu_fw_start()
932 gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid); in a6xx_gmu_fw_start()
934 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG, in a6xx_gmu_fw_start()
935 gmu->log.iova | (gmu->log.size / SZ_4K - 1)); in a6xx_gmu_fw_start()
938 /* Set up the lowest idle level on the GMU */ in a6xx_gmu_fw_start()
939 a6xx_gmu_power_config(gmu); in a6xx_gmu_fw_start()
941 ret = a6xx_gmu_start(gmu); in a6xx_gmu_fw_start()
945 if (gmu->legacy) { in a6xx_gmu_fw_start()
946 ret = a6xx_gmu_gfx_rail_on(gmu); in a6xx_gmu_fw_start()
950 ret = a6xx_sptprac_enable(gmu); in a6xx_gmu_fw_start()
955 ret = a6xx_gmu_hfi_start(gmu); in a6xx_gmu_fw_start()
973 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu) in a6xx_gmu_irq_disable() argument
975 disable_irq(gmu->gmu_irq); in a6xx_gmu_irq_disable()
976 disable_irq(gmu->hfi_irq); in a6xx_gmu_irq_disable()
978 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0); in a6xx_gmu_irq_disable()
979 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0); in a6xx_gmu_irq_disable()
982 static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) in a6xx_gmu_rpmh_off() argument
984 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_rpmh_off()
993 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
995 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
997 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
999 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1005 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS4_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1007 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS5_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1009 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS6_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1011 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS7_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1013 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS8_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1015 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS9_DRV0_STATUS + seqmem_off, in a6xx_gmu_rpmh_off()
1019 /* Force the GMU off in case it isn't responsive */
1020 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) in a6xx_gmu_force_off() argument
1022 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_force_off()
1030 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); in a6xx_gmu_force_off()
1033 a6xx_hfi_stop(gmu); in a6xx_gmu_force_off()
1036 a6xx_gmu_irq_disable(gmu); in a6xx_gmu_force_off()
1038 /* Force off SPTP in case the GMU is managing it */ in a6xx_gmu_force_off()
1039 a6xx_sptprac_disable(gmu); in a6xx_gmu_force_off()
1041 a6xx_gemnoc_workaround(gmu); in a6xx_gmu_force_off()
1044 a6xx_gmu_rpmh_off(gmu); in a6xx_gmu_force_off()
1047 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7); in a6xx_gmu_force_off()
1048 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); in a6xx_gmu_force_off()
1053 /* Halt the gmu cm3 core */ in a6xx_gmu_force_off()
1054 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); in a6xx_gmu_force_off()
1061 a6xx_rpmh_stop(gmu); in a6xx_gmu_force_off()
1064 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) in a6xx_gmu_set_initial_freq() argument
1067 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index]; in a6xx_gmu_set_initial_freq()
1073 gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */ in a6xx_gmu_set_initial_freq()
1078 static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu) in a6xx_gmu_set_initial_bw() argument
1081 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index]; in a6xx_gmu_set_initial_bw()
1095 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_resume() local
1098 if (WARN(!gmu->initialized, "The GMU is not set up yet\n")) in a6xx_gmu_resume()
1101 gmu->hung = false; in a6xx_gmu_resume()
1104 pm_runtime_get_sync(gmu->dev); in a6xx_gmu_resume()
1109 * bring down the GX after a GMU failure in a6xx_gmu_resume()
1111 if (!IS_ERR_OR_NULL(gmu->gxpd)) in a6xx_gmu_resume()
1112 pm_runtime_get_sync(gmu->gxpd); in a6xx_gmu_resume()
1114 /* Use a known rate to bring up the GMU */ in a6xx_gmu_resume()
1115 clk_set_rate(gmu->core_clk, 200000000); in a6xx_gmu_resume()
1116 clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ? in a6xx_gmu_resume()
1118 ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); in a6xx_gmu_resume()
1120 pm_runtime_put(gmu->gxpd); in a6xx_gmu_resume()
1121 pm_runtime_put(gmu->dev); in a6xx_gmu_resume()
1126 a6xx_gmu_set_initial_bw(gpu, gmu); in a6xx_gmu_resume()
1128 /* Enable the GMU interrupt */ in a6xx_gmu_resume()
1129 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0); in a6xx_gmu_resume()
1130 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK); in a6xx_gmu_resume()
1131 enable_irq(gmu->gmu_irq); in a6xx_gmu_resume()
1137 } else if (gmu->legacy) { in a6xx_gmu_resume()
1138 status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ? in a6xx_gmu_resume()
1148 ret = a6xx_gmu_fw_start(gmu, status); in a6xx_gmu_resume()
1152 ret = a6xx_hfi_start(gmu, status); in a6xx_gmu_resume()
1157 * Turn on the GMU firmware fault interrupt after we know the boot in a6xx_gmu_resume()
1160 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0); in a6xx_gmu_resume()
1161 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK); in a6xx_gmu_resume()
1162 enable_irq(gmu->hfi_irq); in a6xx_gmu_resume()
1165 a6xx_gmu_set_initial_freq(gpu, gmu); in a6xx_gmu_resume()
1168 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_resume()
1170 set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status); in a6xx_gmu_resume()
1173 /* On failure, shut down the GMU to leave it in a good state */ in a6xx_gmu_resume()
1175 disable_irq(gmu->gmu_irq); in a6xx_gmu_resume()
1176 a6xx_rpmh_stop(gmu); in a6xx_gmu_resume()
1177 pm_runtime_put(gmu->gxpd); in a6xx_gmu_resume()
1178 pm_runtime_put(gmu->dev); in a6xx_gmu_resume()
1184 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) in a6xx_gmu_isidle() argument
1188 if (!gmu->initialized) in a6xx_gmu_isidle()
1191 reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS); in a6xx_gmu_isidle()
1199 /* Gracefully try to shut down the GMU and by extension the GPU */
1200 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) in a6xx_gmu_shutdown() argument
1202 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_shutdown()
1208 * GMU firmware's internal power state gets messed up if we send "prepare_slumber" hfi when in a6xx_gmu_shutdown()
1213 if (a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET)) in a6xx_gmu_shutdown()
1216 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); in a6xx_gmu_shutdown()
1219 if (test_and_clear_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status)) in a6xx_gmu_shutdown()
1220 a6xx_gmu_clear_oob(gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_shutdown()
1222 ret = a6xx_gmu_wait_for_idle(gmu); in a6xx_gmu_shutdown()
1224 /* If the GMU isn't responding assume it is hung */ in a6xx_gmu_shutdown()
1230 /* tell the GMU we want to slumber */ in a6xx_gmu_shutdown()
1231 ret = a6xx_gmu_notify_slumber(gmu); in a6xx_gmu_shutdown()
1235 ret = gmu_poll_timeout(gmu, in a6xx_gmu_shutdown()
1246 DRM_DEV_ERROR(gmu->dev, in a6xx_gmu_shutdown()
1247 "Unable to slumber GMU: status = 0%x/0%x\n", in a6xx_gmu_shutdown()
1248 gmu_read(gmu, in a6xx_gmu_shutdown()
1250 gmu_read(gmu, in a6xx_gmu_shutdown()
1254 a6xx_hfi_stop(gmu); in a6xx_gmu_shutdown()
1257 a6xx_gmu_irq_disable(gmu); in a6xx_gmu_shutdown()
1260 a6xx_rpmh_stop(gmu); in a6xx_gmu_shutdown()
1265 a6xx_gmu_force_off(gmu); in a6xx_gmu_shutdown()
1271 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_stop() local
1274 if (!pm_runtime_active(gmu->dev)) in a6xx_gmu_stop()
1278 * Force the GMU off if we detected a hang, otherwise try to shut it in a6xx_gmu_stop()
1281 if (gmu->hung) in a6xx_gmu_stop()
1282 a6xx_gmu_force_off(gmu); in a6xx_gmu_stop()
1284 a6xx_gmu_shutdown(gmu); in a6xx_gmu_stop()
1290 * Make sure the GX domain is off before turning off the GMU (CX) in a6xx_gmu_stop()
1291 * domain. Usually the GMU does this but only if the shutdown sequence in a6xx_gmu_stop()
1294 if (!IS_ERR_OR_NULL(gmu->gxpd)) in a6xx_gmu_stop()
1295 pm_runtime_put_sync(gmu->gxpd); in a6xx_gmu_stop()
1297 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); in a6xx_gmu_stop()
1299 pm_runtime_put_sync(gmu->dev); in a6xx_gmu_stop()
1304 static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu) in a6xx_gmu_memory_free() argument
1306 struct msm_mmu *mmu = to_msm_vm(gmu->vm)->mmu; in a6xx_gmu_memory_free()
1308 msm_gem_kernel_put(gmu->hfi.obj, gmu->vm); in a6xx_gmu_memory_free()
1309 msm_gem_kernel_put(gmu->debug.obj, gmu->vm); in a6xx_gmu_memory_free()
1310 msm_gem_kernel_put(gmu->icache.obj, gmu->vm); in a6xx_gmu_memory_free()
1311 msm_gem_kernel_put(gmu->dcache.obj, gmu->vm); in a6xx_gmu_memory_free()
1312 msm_gem_kernel_put(gmu->dummy.obj, gmu->vm); in a6xx_gmu_memory_free()
1313 msm_gem_kernel_put(gmu->log.obj, gmu->vm); in a6xx_gmu_memory_free()
1316 drm_gpuvm_put(gmu->vm); in a6xx_gmu_memory_free()
1319 static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo, in a6xx_gmu_memory_alloc() argument
1322 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_memory_alloc()
1330 /* no fixed address - use GMU's uncached range */ in a6xx_gmu_memory_alloc()
1345 ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->vm, &bo->iova, in a6xx_gmu_memory_alloc()
1360 static int a6xx_gmu_memory_probe(struct drm_device *drm, struct a6xx_gmu *gmu) in a6xx_gmu_memory_probe() argument
1364 mmu = msm_iommu_new(gmu->dev, 0); in a6xx_gmu_memory_probe()
1368 gmu->vm = msm_gem_vm_create(drm, mmu, "gmu", 0x0, 0x80000000, true); in a6xx_gmu_memory_probe()
1369 if (IS_ERR(gmu->vm)) in a6xx_gmu_memory_probe()
1370 return PTR_ERR(gmu->vm); in a6xx_gmu_memory_probe()
1391 struct a6xx_gmu *gmu) in a6xx_gmu_rpmh_bw_votes_init() argument
1410 dev_err(gmu->dev, "invalid BCM '%s' aux data size\n", in a6xx_gmu_rpmh_bw_votes_init()
1419 for (bw_index = 0; bw_index < gmu->nr_gpu_bws; bw_index++) { in a6xx_gmu_rpmh_bw_votes_init()
1420 u32 *data = gmu->gpu_ib_votes[bw_index]; in a6xx_gmu_rpmh_bw_votes_init()
1421 u32 bw = gmu->gpu_bw_table[bw_index]; in a6xx_gmu_rpmh_bw_votes_init()
1443 /* GMU on A6xx votes perfmode on all valid bandwidth */ in a6xx_gmu_rpmh_bw_votes_init()
1573 * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
1576 * The GMU can also vote for DDR interconnects, use the OPP bandwidth entries
1580 static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu) in a6xx_gmu_rpmh_votes_init() argument
1582 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_rpmh_votes_init()
1589 ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes, in a6xx_gmu_rpmh_votes_init()
1590 gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl"); in a6xx_gmu_rpmh_votes_init()
1593 ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes, in a6xx_gmu_rpmh_votes_init()
1594 gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl"); in a6xx_gmu_rpmh_votes_init()
1597 if (info->bcms && gmu->nr_gpu_bws > 1) in a6xx_gmu_rpmh_votes_init()
1598 ret |= a6xx_gmu_rpmh_bw_votes_init(adreno_gpu, info, gmu); in a6xx_gmu_rpmh_votes_init()
1617 "The GMU frequency table is being truncated\n")) in a6xx_gmu_build_freq_table()
1649 "The GMU bandwidth table is being truncated\n")) in a6xx_gmu_build_bw_table()
1667 static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu) in a6xx_gmu_pwrlevels_probe() argument
1669 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_pwrlevels_probe()
1677 * The GMU handles its own frequency switching so build a list of in a6xx_gmu_pwrlevels_probe()
1680 ret = devm_pm_opp_of_add_table(gmu->dev); in a6xx_gmu_pwrlevels_probe()
1682 DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n"); in a6xx_gmu_pwrlevels_probe()
1686 gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev, in a6xx_gmu_pwrlevels_probe()
1687 gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs)); in a6xx_gmu_pwrlevels_probe()
1690 * The GMU also handles GPU frequency switching so build a list in a6xx_gmu_pwrlevels_probe()
1693 gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev, in a6xx_gmu_pwrlevels_probe()
1694 gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs)); in a6xx_gmu_pwrlevels_probe()
1696 gmu->current_perf_index = gmu->nr_gpu_freqs - 1; in a6xx_gmu_pwrlevels_probe()
1699 * The GMU also handles GPU Interconnect Votes so build a list in a6xx_gmu_pwrlevels_probe()
1703 gmu->nr_gpu_bws = a6xx_gmu_build_bw_table(&gpu->pdev->dev, in a6xx_gmu_pwrlevels_probe()
1704 gmu->gpu_bw_table, ARRAY_SIZE(gmu->gpu_bw_table)); in a6xx_gmu_pwrlevels_probe()
1706 /* Build the list of RPMh votes that we'll send to the GMU */ in a6xx_gmu_pwrlevels_probe()
1707 return a6xx_gmu_rpmh_votes_init(gmu); in a6xx_gmu_pwrlevels_probe()
1710 static int a6xx_gmu_acd_probe(struct a6xx_gmu *gmu) in a6xx_gmu_acd_probe() argument
1712 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); in a6xx_gmu_acd_probe()
1713 struct a6xx_hfi_acd_table *cmd = &gmu->acd_table; in a6xx_gmu_acd_probe()
1721 DRM_DEV_ERROR(gmu->dev, "Skipping GPU ACD probe\n"); in a6xx_gmu_acd_probe()
1730 for (i = 1; i < gmu->nr_gpu_freqs; i++) { in a6xx_gmu_acd_probe()
1736 freq = gmu->gpu_freqs[i]; in a6xx_gmu_acd_probe()
1747 DRM_DEV_ERROR(gmu->dev, "Unable to read acd level for freq %lu\n", freq); in a6xx_gmu_acd_probe()
1758 if (cmd->enable_by_level && IS_ERR_OR_NULL(gmu->qmp)) { in a6xx_gmu_acd_probe()
1759 DRM_DEV_ERROR(gmu->dev, "Unable to send ACD state to AOSS\n"); in a6xx_gmu_acd_probe()
1764 if (IS_ERR_OR_NULL(gmu->qmp)) in a6xx_gmu_acd_probe()
1771 ret = qmp_send(gmu->qmp, "{class: gpu, res: acd, val: %d}", !!cmd->enable_by_level); in a6xx_gmu_acd_probe()
1773 DRM_DEV_ERROR(gmu->dev, "Failed to send ACD state to AOSS\n"); in a6xx_gmu_acd_probe()
1780 static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) in a6xx_gmu_clocks_probe() argument
1782 int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks); in a6xx_gmu_clocks_probe()
1787 gmu->nr_clocks = ret; in a6xx_gmu_clocks_probe()
1789 gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks, in a6xx_gmu_clocks_probe()
1790 gmu->nr_clocks, "gmu"); in a6xx_gmu_clocks_probe()
1792 gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks, in a6xx_gmu_clocks_probe()
1793 gmu->nr_clocks, "hub"); in a6xx_gmu_clocks_probe()
1819 static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, in a6xx_gmu_get_irq() argument
1826 ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN, name, gmu); in a6xx_gmu_get_irq()
1840 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_sysprof_setup() local
1843 /* Nothing to do if GPU is suspended. We will handle this during GMU resume */ in a6xx_gmu_sysprof_setup()
1847 mutex_lock(&gmu->lock); in a6xx_gmu_sysprof_setup()
1855 if ((sysprof_active > 1) && !test_and_set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status)) in a6xx_gmu_sysprof_setup()
1856 a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_sysprof_setup()
1857 else if ((sysprof_active == 1) && test_and_clear_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status)) in a6xx_gmu_sysprof_setup()
1858 a6xx_gmu_clear_oob(gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_sysprof_setup()
1860 mutex_unlock(&gmu->lock); in a6xx_gmu_sysprof_setup()
1868 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_remove() local
1869 struct platform_device *pdev = to_platform_device(gmu->dev); in a6xx_gmu_remove()
1871 mutex_lock(&gmu->lock); in a6xx_gmu_remove()
1872 if (!gmu->initialized) { in a6xx_gmu_remove()
1873 mutex_unlock(&gmu->lock); in a6xx_gmu_remove()
1877 gmu->initialized = false; in a6xx_gmu_remove()
1879 mutex_unlock(&gmu->lock); in a6xx_gmu_remove()
1881 pm_runtime_force_suspend(gmu->dev); in a6xx_gmu_remove()
1884 * Since cxpd is a virt device, the devlink with gmu-dev will be removed in a6xx_gmu_remove()
1887 dev_pm_domain_detach(gmu->cxpd, false); in a6xx_gmu_remove()
1889 if (!IS_ERR_OR_NULL(gmu->gxpd)) { in a6xx_gmu_remove()
1890 pm_runtime_disable(gmu->gxpd); in a6xx_gmu_remove()
1891 dev_pm_domain_detach(gmu->gxpd, false); in a6xx_gmu_remove()
1894 if (!IS_ERR_OR_NULL(gmu->qmp)) in a6xx_gmu_remove()
1895 qmp_put(gmu->qmp); in a6xx_gmu_remove()
1897 iounmap(gmu->mmio); in a6xx_gmu_remove()
1899 iounmap(gmu->rscc); in a6xx_gmu_remove()
1900 gmu->mmio = NULL; in a6xx_gmu_remove()
1901 gmu->rscc = NULL; in a6xx_gmu_remove()
1904 a6xx_gmu_memory_free(gmu); in a6xx_gmu_remove()
1906 free_irq(gmu->gmu_irq, gmu); in a6xx_gmu_remove()
1907 free_irq(gmu->hfi_irq, gmu); in a6xx_gmu_remove()
1911 put_device(gmu->dev); in a6xx_gmu_remove()
1917 struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb); in cxpd_notifier_cb() local
1920 complete_all(&gmu->pd_gate); in cxpd_notifier_cb()
1928 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_wrapper_init() local
1934 gmu->dev = &pdev->dev; in a6xx_gmu_wrapper_init()
1936 ret = of_dma_configure(gmu->dev, node, true); in a6xx_gmu_wrapper_init()
1940 pm_runtime_enable(gmu->dev); in a6xx_gmu_wrapper_init()
1943 gmu->legacy = true; in a6xx_gmu_wrapper_init()
1945 /* Map the GMU registers */ in a6xx_gmu_wrapper_init()
1946 gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); in a6xx_gmu_wrapper_init()
1947 if (IS_ERR(gmu->mmio)) { in a6xx_gmu_wrapper_init()
1948 ret = PTR_ERR(gmu->mmio); in a6xx_gmu_wrapper_init()
1952 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); in a6xx_gmu_wrapper_init()
1953 if (IS_ERR(gmu->cxpd)) { in a6xx_gmu_wrapper_init()
1954 ret = PTR_ERR(gmu->cxpd); in a6xx_gmu_wrapper_init()
1958 if (!device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME)) { in a6xx_gmu_wrapper_init()
1963 init_completion(&gmu->pd_gate); in a6xx_gmu_wrapper_init()
1964 complete_all(&gmu->pd_gate); in a6xx_gmu_wrapper_init()
1965 gmu->pd_nb.notifier_call = cxpd_notifier_cb; in a6xx_gmu_wrapper_init()
1968 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx"); in a6xx_gmu_wrapper_init()
1969 if (IS_ERR(gmu->gxpd)) { in a6xx_gmu_wrapper_init()
1970 ret = PTR_ERR(gmu->gxpd); in a6xx_gmu_wrapper_init()
1974 gmu->initialized = true; in a6xx_gmu_wrapper_init()
1979 dev_pm_domain_detach(gmu->cxpd, false); in a6xx_gmu_wrapper_init()
1982 iounmap(gmu->mmio); in a6xx_gmu_wrapper_init()
1985 put_device(gmu->dev); in a6xx_gmu_wrapper_init()
1993 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_gmu_init() local
2001 gmu->dev = &pdev->dev; in a6xx_gmu_init()
2003 ret = of_dma_configure(gmu->dev, node, true); in a6xx_gmu_init()
2007 /* Set GMU idle level */ in a6xx_gmu_init()
2008 gmu->idle_level = (adreno_gpu->info->quirks & ADRENO_QUIRK_IFPC) ? in a6xx_gmu_init()
2011 pm_runtime_enable(gmu->dev); in a6xx_gmu_init()
2014 ret = a6xx_gmu_clocks_probe(gmu); in a6xx_gmu_init()
2018 ret = a6xx_gmu_memory_probe(adreno_gpu->base.dev, gmu); in a6xx_gmu_init()
2023 /* A660 now requires handling "prealloc requests" in GMU firmware in a6xx_gmu_init()
2029 gmu->dummy.size = SZ_4K; in a6xx_gmu_init()
2032 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7, in a6xx_gmu_init()
2037 gmu->dummy.size = SZ_8K; in a6xx_gmu_init()
2040 /* Allocate memory for the GMU dummy page */ in a6xx_gmu_init()
2041 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, gmu->dummy.size, in a6xx_gmu_init()
2049 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, in a6xx_gmu_init()
2060 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, in a6xx_gmu_init()
2065 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache, in a6xx_gmu_init()
2071 gmu->legacy = true; in a6xx_gmu_init()
2073 /* Allocate memory for the GMU debug region */ in a6xx_gmu_init()
2074 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0, "debug"); in a6xx_gmu_init()
2079 /* Allocate memory for the GMU log region */ in a6xx_gmu_init()
2080 ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_16K, 0, "log"); in a6xx_gmu_init()
2085 ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0, "hfi"); in a6xx_gmu_init()
2089 /* Map the GMU registers */ in a6xx_gmu_init()
2090 gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); in a6xx_gmu_init()
2091 if (IS_ERR(gmu->mmio)) { in a6xx_gmu_init()
2092 ret = PTR_ERR(gmu->mmio); in a6xx_gmu_init()
2098 gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc"); in a6xx_gmu_init()
2099 if (IS_ERR(gmu->rscc)) { in a6xx_gmu_init()
2104 gmu->rscc = gmu->mmio + 0x23000; in a6xx_gmu_init()
2107 /* Get the HFI and GMU interrupts */ in a6xx_gmu_init()
2108 gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq); in a6xx_gmu_init()
2109 gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq); in a6xx_gmu_init()
2111 if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) { in a6xx_gmu_init()
2116 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); in a6xx_gmu_init()
2117 if (IS_ERR(gmu->cxpd)) { in a6xx_gmu_init()
2118 ret = PTR_ERR(gmu->cxpd); in a6xx_gmu_init()
2122 link = device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME); in a6xx_gmu_init()
2129 gmu->qmp = qmp_get(gmu->dev); in a6xx_gmu_init()
2130 if (PTR_ERR_OR_ZERO(gmu->qmp) == -EPROBE_DEFER) { in a6xx_gmu_init()
2135 init_completion(&gmu->pd_gate); in a6xx_gmu_init()
2136 complete_all(&gmu->pd_gate); in a6xx_gmu_init()
2137 gmu->pd_nb.notifier_call = cxpd_notifier_cb; in a6xx_gmu_init()
2140 * Get a link to the GX power domain to reset the GPU in case of GMU in a6xx_gmu_init()
2143 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx"); in a6xx_gmu_init()
2145 /* Get the power levels for the GMU and GPU */ in a6xx_gmu_init()
2146 a6xx_gmu_pwrlevels_probe(gmu); in a6xx_gmu_init()
2148 ret = a6xx_gmu_acd_probe(gmu); in a6xx_gmu_init()
2153 a6xx_hfi_init(gmu); in a6xx_gmu_init()
2156 a6xx_gmu_rpmh_init(gmu); in a6xx_gmu_init()
2158 gmu->initialized = true; in a6xx_gmu_init()
2163 if (!IS_ERR_OR_NULL(gmu->gxpd)) in a6xx_gmu_init()
2164 dev_pm_domain_detach(gmu->gxpd, false); in a6xx_gmu_init()
2166 if (!IS_ERR_OR_NULL(gmu->qmp)) in a6xx_gmu_init()
2167 qmp_put(gmu->qmp); in a6xx_gmu_init()
2172 dev_pm_domain_detach(gmu->cxpd, false); in a6xx_gmu_init()
2175 iounmap(gmu->mmio); in a6xx_gmu_init()
2177 iounmap(gmu->rscc); in a6xx_gmu_init()
2178 free_irq(gmu->gmu_irq, gmu); in a6xx_gmu_init()
2179 free_irq(gmu->hfi_irq, gmu); in a6xx_gmu_init()
2182 a6xx_gmu_memory_free(gmu); in a6xx_gmu_init()
2185 put_device(gmu->dev); in a6xx_gmu_init()