1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/clk.h> 6 #include <linux/interconnect.h> 7 #include <linux/of_platform.h> 8 #include <linux/platform_device.h> 9 #include <linux/pm_domain.h> 10 #include <linux/pm_opp.h> 11 #include <soc/qcom/cmd-db.h> 12 #include <drm/drm_gem.h> 13 14 #include "a6xx_gpu.h" 15 #include "a6xx_gmu.xml.h" 16 #include "msm_gem.h" 17 #include "msm_gpu_trace.h" 18 #include "msm_mmu.h" 19 20 static void a6xx_gmu_fault(struct a6xx_gmu *gmu) 21 { 22 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 23 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 24 struct msm_gpu *gpu = &adreno_gpu->base; 25 26 /* FIXME: add a banner here */ 27 gmu->hung = true; 28 29 /* Turn off the hangcheck timer while we are resetting */ 30 del_timer(&gpu->hangcheck_timer); 31 32 /* Queue the GPU handler because we need to treat this as a recovery */ 33 kthread_queue_work(gpu->worker, &gpu->recover_work); 34 } 35 36 static irqreturn_t a6xx_gmu_irq(int irq, void *data) 37 { 38 struct a6xx_gmu *gmu = data; 39 u32 status; 40 41 status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS); 42 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status); 43 44 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) { 45 dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n"); 46 47 a6xx_gmu_fault(gmu); 48 } 49 50 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR) 51 dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n"); 52 53 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR) 54 dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n", 55 gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS)); 56 57 return IRQ_HANDLED; 58 } 59 60 static irqreturn_t a6xx_hfi_irq(int irq, void *data) 61 { 62 struct a6xx_gmu *gmu = data; 63 u32 status; 64 65 status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO); 66 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status); 67 68 if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) { 69 dev_err_ratelimited(gmu->dev, "GMU firmware fault\n"); 70 71 a6xx_gmu_fault(gmu); 72 } 73 74 return IRQ_HANDLED; 75 } 76 77 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu) 78 { 79 u32 val; 80 81 /* This can be called from gpu state code so make sure GMU is valid */ 82 if (!gmu->initialized) 83 return false; 84 85 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); 86 87 return !(val & 88 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF | 89 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF)); 90 } 91 92 /* Check to see if the GX rail is still powered */ 93 bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu) 94 { 95 u32 val; 96 97 /* This can be called from gpu state code so make sure GMU is valid */ 98 if (!gmu->initialized) 99 return false; 100 101 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); 102 103 return !(val & 104 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF | 105 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF)); 106 } 107 108 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, 109 bool suspended) 110 { 111 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 112 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 113 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 114 u32 perf_index; 115 unsigned long gpu_freq; 116 int ret = 0; 117 118 gpu_freq = dev_pm_opp_get_freq(opp); 119 120 if (gpu_freq == gmu->freq) 121 return; 122 123 for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++) 124 if (gpu_freq == gmu->gpu_freqs[perf_index]) 125 break; 126 127 gmu->current_perf_index = perf_index; 128 gmu->freq = gmu->gpu_freqs[perf_index]; 129 130 trace_msm_gmu_freq_change(gmu->freq, perf_index); 131 132 /* 133 * This can get called from devfreq while the hardware is idle. Don't 134 * bring up the power if it isn't already active. All we're doing here 135 * is updating the frequency so that when we come back online we're at 136 * the right rate. 137 */ 138 if (suspended) 139 return; 140 141 if (!gmu->legacy) { 142 a6xx_hfi_set_freq(gmu, perf_index); 143 dev_pm_opp_set_opp(&gpu->pdev->dev, opp); 144 return; 145 } 146 147 gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0); 148 149 gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING, 150 ((3 & 0xf) << 28) | perf_index); 151 152 /* 153 * Send an invalid index as a vote for the bus bandwidth and let the 154 * firmware decide on the right vote 155 */ 156 gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff); 157 158 /* Set and clear the OOB for DCVS to trigger the GMU */ 159 a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET); 160 a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET); 161 162 ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN); 163 if (ret) 164 dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret); 165 166 dev_pm_opp_set_opp(&gpu->pdev->dev, opp); 167 } 168 169 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu) 170 { 171 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 172 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 173 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 174 175 return gmu->freq; 176 } 177 178 static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu) 179 { 180 u32 val; 181 int local = gmu->idle_level; 182 183 /* SPTP and IFPC both report as IFPC */ 184 if (gmu->idle_level == GMU_IDLE_STATE_SPTP) 185 local = GMU_IDLE_STATE_IFPC; 186 187 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE); 188 189 if (val == local) { 190 if (gmu->idle_level != GMU_IDLE_STATE_IFPC || 191 !a6xx_gmu_gx_is_on(gmu)) 192 return true; 193 } 194 195 return false; 196 } 197 198 /* Wait for the GMU to get to its most idle state */ 199 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu) 200 { 201 return spin_until(a6xx_gmu_check_idle_level(gmu)); 202 } 203 204 static int a6xx_gmu_start(struct a6xx_gmu *gmu) 205 { 206 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 207 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 208 u32 mask, reset_val, val; 209 int ret; 210 211 val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8); 212 if (val <= 0x20010004) { 213 mask = 0xffffffff; 214 reset_val = 0xbabeface; 215 } else { 216 mask = 0x1ff; 217 reset_val = 0x100; 218 } 219 220 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); 221 222 /* Set the log wptr index 223 * note: downstream saves the value in poweroff and restores it here 224 */ 225 if (adreno_is_a7xx(adreno_gpu)) 226 gmu_write(gmu, REG_A7XX_GMU_GENERAL_9, 0); 227 else 228 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0); 229 230 231 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0); 232 233 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val, 234 (val & mask) == reset_val, 100, 10000); 235 236 if (ret) 237 DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n"); 238 239 return ret; 240 } 241 242 static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu) 243 { 244 u32 val; 245 int ret; 246 247 gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1); 248 249 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val, 250 val & 1, 100, 10000); 251 if (ret) 252 DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n"); 253 254 return ret; 255 } 256 257 struct a6xx_gmu_oob_bits { 258 int set, ack, set_new, ack_new, clear, clear_new; 259 const char *name; 260 }; 261 262 /* These are the interrupt / ack bits for each OOB request that are set 263 * in a6xx_gmu_set_oob and a6xx_clear_oob 264 */ 265 static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = { 266 [GMU_OOB_GPU_SET] = { 267 .name = "GPU_SET", 268 .set = 16, 269 .ack = 24, 270 .set_new = 30, 271 .ack_new = 31, 272 .clear = 24, 273 .clear_new = 31, 274 }, 275 276 [GMU_OOB_PERFCOUNTER_SET] = { 277 .name = "PERFCOUNTER", 278 .set = 17, 279 .ack = 25, 280 .set_new = 28, 281 .ack_new = 30, 282 .clear = 25, 283 .clear_new = 29, 284 }, 285 286 [GMU_OOB_BOOT_SLUMBER] = { 287 .name = "BOOT_SLUMBER", 288 .set = 22, 289 .ack = 30, 290 .clear = 30, 291 }, 292 293 [GMU_OOB_DCVS_SET] = { 294 .name = "GPU_DCVS", 295 .set = 23, 296 .ack = 31, 297 .clear = 31, 298 }, 299 }; 300 301 /* Trigger a OOB (out of band) request to the GMU */ 302 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) 303 { 304 int ret; 305 u32 val; 306 int request, ack; 307 308 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); 309 310 if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) 311 return -EINVAL; 312 313 if (gmu->legacy) { 314 request = a6xx_gmu_oob_bits[state].set; 315 ack = a6xx_gmu_oob_bits[state].ack; 316 } else { 317 request = a6xx_gmu_oob_bits[state].set_new; 318 ack = a6xx_gmu_oob_bits[state].ack_new; 319 if (!request || !ack) { 320 DRM_DEV_ERROR(gmu->dev, 321 "Invalid non-legacy GMU request %s\n", 322 a6xx_gmu_oob_bits[state].name); 323 return -EINVAL; 324 } 325 } 326 327 /* Trigger the equested OOB operation */ 328 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request); 329 330 /* Wait for the acknowledge interrupt */ 331 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val, 332 val & (1 << ack), 100, 10000); 333 334 if (ret) 335 DRM_DEV_ERROR(gmu->dev, 336 "Timeout waiting for GMU OOB set %s: 0x%x\n", 337 a6xx_gmu_oob_bits[state].name, 338 gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO)); 339 340 /* Clear the acknowledge interrupt */ 341 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack); 342 343 return ret; 344 } 345 346 /* Clear a pending OOB state in the GMU */ 347 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) 348 { 349 int bit; 350 351 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock)); 352 353 if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits)) 354 return; 355 356 if (gmu->legacy) 357 bit = a6xx_gmu_oob_bits[state].clear; 358 else 359 bit = a6xx_gmu_oob_bits[state].clear_new; 360 361 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit); 362 } 363 364 /* Enable CPU control of SPTP power power collapse */ 365 int a6xx_sptprac_enable(struct a6xx_gmu *gmu) 366 { 367 int ret; 368 u32 val; 369 370 if (!gmu->legacy) 371 return 0; 372 373 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000); 374 375 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, 376 (val & 0x38) == 0x28, 1, 100); 377 378 if (ret) { 379 DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n", 380 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); 381 } 382 383 return 0; 384 } 385 386 /* Disable CPU control of SPTP power power collapse */ 387 void a6xx_sptprac_disable(struct a6xx_gmu *gmu) 388 { 389 u32 val; 390 int ret; 391 392 if (!gmu->legacy) 393 return; 394 395 /* Make sure retention is on */ 396 gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11)); 397 398 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001); 399 400 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, 401 (val & 0x04), 100, 10000); 402 403 if (ret) 404 DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n", 405 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); 406 } 407 408 /* Let the GMU know we are starting a boot sequence */ 409 static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu) 410 { 411 u32 vote; 412 413 /* Let the GMU know we are getting ready for boot */ 414 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0); 415 416 /* Choose the "default" power level as the highest available */ 417 vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1]; 418 419 gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff); 420 gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff); 421 422 /* Let the GMU know the boot sequence has started */ 423 return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); 424 } 425 426 /* Let the GMU know that we are about to go into slumber */ 427 static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) 428 { 429 int ret; 430 431 /* Disable the power counter so the GMU isn't busy */ 432 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); 433 434 /* Disable SPTP_PC if the CPU is responsible for it */ 435 if (gmu->idle_level < GMU_IDLE_STATE_SPTP) 436 a6xx_sptprac_disable(gmu); 437 438 if (!gmu->legacy) { 439 ret = a6xx_hfi_send_prep_slumber(gmu); 440 goto out; 441 } 442 443 /* Tell the GMU to get ready to slumber */ 444 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1); 445 446 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); 447 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER); 448 449 if (!ret) { 450 /* Check to see if the GMU really did slumber */ 451 if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE) 452 != 0x0f) { 453 DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n"); 454 ret = -ETIMEDOUT; 455 } 456 } 457 458 out: 459 /* Put fence into allow mode */ 460 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); 461 return ret; 462 } 463 464 static int a6xx_rpmh_start(struct a6xx_gmu *gmu) 465 { 466 int ret; 467 u32 val; 468 469 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, BIT(1)); 470 471 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val, 472 val & (1 << 1), 100, 10000); 473 if (ret) { 474 DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n"); 475 return ret; 476 } 477 478 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val, 479 !val, 100, 10000); 480 481 if (ret) { 482 DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n"); 483 return ret; 484 } 485 486 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); 487 488 return 0; 489 } 490 491 static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) 492 { 493 int ret; 494 u32 val; 495 496 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); 497 498 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, 499 val, val & (1 << 16), 100, 10000); 500 if (ret) 501 DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n"); 502 503 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); 504 } 505 506 static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value) 507 { 508 writel(value, ptr + (offset << 2)); 509 } 510 511 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, 512 const char *name); 513 514 static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) 515 { 516 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 517 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 518 struct platform_device *pdev = to_platform_device(gmu->dev); 519 void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc"); 520 u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0; 521 void __iomem *seqptr = NULL; 522 uint32_t pdc_address_offset; 523 bool pdc_in_aop = false; 524 525 if (IS_ERR(pdcptr)) 526 goto err; 527 528 if (adreno_is_a650(adreno_gpu) || 529 adreno_is_a660_family(adreno_gpu) || 530 adreno_is_a7xx(adreno_gpu)) 531 pdc_in_aop = true; 532 else if (adreno_is_a618(adreno_gpu) || adreno_is_a640_family(adreno_gpu)) 533 pdc_address_offset = 0x30090; 534 else if (adreno_is_a619(adreno_gpu)) 535 pdc_address_offset = 0x300a0; 536 else 537 pdc_address_offset = 0x30080; 538 539 if (!pdc_in_aop) { 540 seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq"); 541 if (IS_ERR(seqptr)) 542 goto err; 543 } 544 545 /* Disable SDE clock gating */ 546 gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24)); 547 548 /* Setup RSC PDC handshake for sleep and wakeup */ 549 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1); 550 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0); 551 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0); 552 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0); 553 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0); 554 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 555 adreno_is_a740_family(adreno_gpu) ? 0x80000021 : 0x80000000); 556 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0); 557 gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0); 558 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520); 559 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510); 560 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514); 561 562 /* The second spin of A7xx GPUs messed with some register offsets.. */ 563 if (adreno_is_a740_family(adreno_gpu)) 564 seqmem0_drv0_reg = REG_A7XX_RSCC_SEQ_MEM_0_DRV0_A740; 565 566 /* Load RSC sequencer uCode for sleep and wakeup */ 567 if (adreno_is_a650_family(adreno_gpu) || 568 adreno_is_a7xx(adreno_gpu)) { 569 gmu_write_rscc(gmu, seqmem0_drv0_reg, 0xeaaae5a0); 570 gmu_write_rscc(gmu, seqmem0_drv0_reg + 1, 0xe1a1ebab); 571 gmu_write_rscc(gmu, seqmem0_drv0_reg + 2, 0xa2e0a581); 572 gmu_write_rscc(gmu, seqmem0_drv0_reg + 3, 0xecac82e2); 573 gmu_write_rscc(gmu, seqmem0_drv0_reg + 4, 0x0020edad); 574 } else { 575 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0); 576 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7); 577 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1); 578 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2); 579 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8); 580 } 581 582 if (pdc_in_aop) 583 goto setup_pdc; 584 585 /* Load PDC sequencer uCode for power up and power down sequence */ 586 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1); 587 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2); 588 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0); 589 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284); 590 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc); 591 592 /* Set TCS commands used by PDC sequence for low power modes */ 593 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7); 594 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0); 595 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0); 596 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108); 597 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010); 598 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1); 599 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108); 600 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000); 601 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0); 602 603 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108); 604 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset); 605 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0); 606 607 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7); 608 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0); 609 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0); 610 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108); 611 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010); 612 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2); 613 614 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108); 615 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000); 616 if (adreno_is_a618(adreno_gpu) || adreno_is_a619(adreno_gpu) || 617 adreno_is_a650_family(adreno_gpu)) 618 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2); 619 else 620 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3); 621 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108); 622 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset); 623 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3); 624 625 /* Setup GPU PDC */ 626 setup_pdc: 627 pdc_write(pdcptr, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0); 628 pdc_write(pdcptr, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001); 629 630 /* ensure no writes happen before the uCode is fully written */ 631 wmb(); 632 633 a6xx_rpmh_stop(gmu); 634 635 err: 636 if (!IS_ERR_OR_NULL(pdcptr)) 637 iounmap(pdcptr); 638 if (!IS_ERR_OR_NULL(seqptr)) 639 iounmap(seqptr); 640 } 641 642 /* 643 * The lowest 16 bits of this value are the number of XO clock cycles for main 644 * hysteresis which is set at 0x1680 cycles (300 us). The higher 16 bits are 645 * for the shorter hysteresis that happens after main - this is 0xa (.5 us) 646 */ 647 648 #define GMU_PWR_COL_HYST 0x000a1680 649 650 /* Set up the idle state for the GMU */ 651 static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) 652 { 653 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 654 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 655 656 /* Disable GMU WB/RB buffer */ 657 gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1); 658 gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1); 659 gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1); 660 661 /* A7xx knows better by default! */ 662 if (adreno_is_a7xx(adreno_gpu)) 663 return; 664 665 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400); 666 667 switch (gmu->idle_level) { 668 case GMU_IDLE_STATE_IFPC: 669 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST, 670 GMU_PWR_COL_HYST); 671 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, 672 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE | 673 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE); 674 fallthrough; 675 case GMU_IDLE_STATE_SPTP: 676 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST, 677 GMU_PWR_COL_HYST); 678 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, 679 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE | 680 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE); 681 } 682 683 /* Enable RPMh GPU client */ 684 gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0, 685 A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE | 686 A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE | 687 A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE | 688 A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE | 689 A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE | 690 A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE); 691 } 692 693 struct block_header { 694 u32 addr; 695 u32 size; 696 u32 type; 697 u32 value; 698 u32 data[]; 699 }; 700 701 static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk) 702 { 703 if (!in_range(blk->addr, bo->iova, bo->size)) 704 return false; 705 706 memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size); 707 return true; 708 } 709 710 static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu) 711 { 712 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 713 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 714 const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU]; 715 const struct block_header *blk; 716 u32 reg_offset; 717 718 u32 itcm_base = 0x00000000; 719 u32 dtcm_base = 0x00040000; 720 721 if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) 722 dtcm_base = 0x10004000; 723 724 if (gmu->legacy) { 725 /* Sanity check the size of the firmware that was loaded */ 726 if (fw_image->size > 0x8000) { 727 DRM_DEV_ERROR(gmu->dev, 728 "GMU firmware is bigger than the available region\n"); 729 return -EINVAL; 730 } 731 732 gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START, 733 (u32*) fw_image->data, fw_image->size); 734 return 0; 735 } 736 737 738 for (blk = (const struct block_header *) fw_image->data; 739 (const u8*) blk < fw_image->data + fw_image->size; 740 blk = (const struct block_header *) &blk->data[blk->size >> 2]) { 741 if (blk->size == 0) 742 continue; 743 744 if (in_range(blk->addr, itcm_base, SZ_16K)) { 745 reg_offset = (blk->addr - itcm_base) >> 2; 746 gmu_write_bulk(gmu, 747 REG_A6XX_GMU_CM3_ITCM_START + reg_offset, 748 blk->data, blk->size); 749 } else if (in_range(blk->addr, dtcm_base, SZ_16K)) { 750 reg_offset = (blk->addr - dtcm_base) >> 2; 751 gmu_write_bulk(gmu, 752 REG_A6XX_GMU_CM3_DTCM_START + reg_offset, 753 blk->data, blk->size); 754 } else if (!fw_block_mem(&gmu->icache, blk) && 755 !fw_block_mem(&gmu->dcache, blk) && 756 !fw_block_mem(&gmu->dummy, blk)) { 757 DRM_DEV_ERROR(gmu->dev, 758 "failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n", 759 blk->addr, blk->size, blk->data[0]); 760 } 761 } 762 763 return 0; 764 } 765 766 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) 767 { 768 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 769 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 770 const struct a6xx_info *a6xx_info = adreno_gpu->info->a6xx; 771 u32 fence_range_lower, fence_range_upper; 772 u32 chipid = 0; 773 int ret; 774 775 /* Vote veto for FAL10 */ 776 if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) { 777 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1); 778 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1); 779 } 780 781 /* Turn on TCM (Tightly Coupled Memory) retention */ 782 if (adreno_is_a7xx(adreno_gpu)) 783 a6xx_llc_write(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL, 1); 784 else 785 gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); 786 787 if (state == GMU_WARM_BOOT) { 788 ret = a6xx_rpmh_start(gmu); 789 if (ret) 790 return ret; 791 } else { 792 if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU], 793 "GMU firmware is not loaded\n")) 794 return -ENOENT; 795 796 ret = a6xx_rpmh_start(gmu); 797 if (ret) 798 return ret; 799 800 ret = a6xx_gmu_fw_load(gmu); 801 if (ret) 802 return ret; 803 } 804 805 /* Clear init result to make sure we are getting a fresh value */ 806 gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0); 807 gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02); 808 809 /* Write the iova of the HFI table */ 810 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova); 811 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1); 812 813 if (adreno_is_a7xx(adreno_gpu)) { 814 fence_range_upper = 0x32; 815 fence_range_lower = 0x8a0; 816 } else { 817 fence_range_upper = 0xa; 818 fence_range_lower = 0xa0; 819 } 820 821 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0, 822 BIT(31) | 823 FIELD_PREP(GENMASK(30, 18), fence_range_upper) | 824 FIELD_PREP(GENMASK(17, 0), fence_range_lower)); 825 826 /* 827 * Snapshots toggle the NMI bit which will result in a jump to the NMI 828 * handler instead of __main. Set the M3 config value to avoid that. 829 */ 830 gmu_write(gmu, REG_A6XX_GMU_CM3_CFG, 0x4052); 831 832 if (a6xx_info->gmu_chipid) { 833 chipid = a6xx_info->gmu_chipid; 834 } else { 835 /* 836 * Note that the GMU has a slightly different layout for 837 * chip_id, for whatever reason, so a bit of massaging 838 * is needed. The upper 16b are the same, but minor and 839 * patchid are packed in four bits each with the lower 840 * 8b unused: 841 */ 842 chipid = adreno_gpu->chip_id & 0xffff0000; 843 chipid |= (adreno_gpu->chip_id << 4) & 0xf000; /* minor */ 844 chipid |= (adreno_gpu->chip_id << 8) & 0x0f00; /* patchid */ 845 } 846 847 if (adreno_is_a7xx(adreno_gpu)) { 848 gmu_write(gmu, REG_A7XX_GMU_GENERAL_10, chipid); 849 gmu_write(gmu, REG_A7XX_GMU_GENERAL_8, 850 (gmu->log.iova & GENMASK(31, 12)) | 851 ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0))); 852 } else { 853 gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid); 854 855 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG, 856 gmu->log.iova | (gmu->log.size / SZ_4K - 1)); 857 } 858 859 /* Set up the lowest idle level on the GMU */ 860 a6xx_gmu_power_config(gmu); 861 862 ret = a6xx_gmu_start(gmu); 863 if (ret) 864 return ret; 865 866 if (gmu->legacy) { 867 ret = a6xx_gmu_gfx_rail_on(gmu); 868 if (ret) 869 return ret; 870 } 871 872 /* Enable SPTP_PC if the CPU is responsible for it */ 873 if (gmu->idle_level < GMU_IDLE_STATE_SPTP) { 874 ret = a6xx_sptprac_enable(gmu); 875 if (ret) 876 return ret; 877 } 878 879 ret = a6xx_gmu_hfi_start(gmu); 880 if (ret) 881 return ret; 882 883 /* FIXME: Do we need this wmb() here? */ 884 wmb(); 885 886 return 0; 887 } 888 889 #define A6XX_HFI_IRQ_MASK \ 890 (A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) 891 892 #define A6XX_GMU_IRQ_MASK \ 893 (A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \ 894 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \ 895 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR) 896 897 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu) 898 { 899 disable_irq(gmu->gmu_irq); 900 disable_irq(gmu->hfi_irq); 901 902 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0); 903 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0); 904 } 905 906 static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) 907 { 908 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 909 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 910 u32 val, seqmem_off = 0; 911 912 /* The second spin of A7xx GPUs messed with some register offsets.. */ 913 if (adreno_is_a740_family(adreno_gpu)) 914 seqmem_off = 4; 915 916 /* Make sure there are no outstanding RPMh votes */ 917 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS + seqmem_off, 918 val, (val & 1), 100, 10000); 919 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS + seqmem_off, 920 val, (val & 1), 100, 10000); 921 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS + seqmem_off, 922 val, (val & 1), 100, 10000); 923 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS + seqmem_off, 924 val, (val & 1), 100, 1000); 925 } 926 927 /* Force the GMU off in case it isn't responsive */ 928 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) 929 { 930 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 931 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 932 struct msm_gpu *gpu = &adreno_gpu->base; 933 934 /* 935 * Turn off keep alive that might have been enabled by the hang 936 * interrupt 937 */ 938 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); 939 940 /* Flush all the queues */ 941 a6xx_hfi_stop(gmu); 942 943 /* Stop the interrupts */ 944 a6xx_gmu_irq_disable(gmu); 945 946 /* Force off SPTP in case the GMU is managing it */ 947 a6xx_sptprac_disable(gmu); 948 949 /* Make sure there are no outstanding RPMh votes */ 950 a6xx_gmu_rpmh_off(gmu); 951 952 /* Clear the WRITEDROPPED fields and put fence into allow mode */ 953 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7); 954 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); 955 956 /* Make sure the above writes go through */ 957 wmb(); 958 959 /* Halt the gmu cm3 core */ 960 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); 961 962 a6xx_bus_clear_pending_transactions(adreno_gpu, true); 963 964 /* Reset GPU core blocks */ 965 a6xx_gpu_sw_reset(gpu, true); 966 } 967 968 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) 969 { 970 struct dev_pm_opp *gpu_opp; 971 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index]; 972 973 gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true); 974 if (IS_ERR(gpu_opp)) 975 return; 976 977 gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */ 978 a6xx_gmu_set_freq(gpu, gpu_opp, false); 979 dev_pm_opp_put(gpu_opp); 980 } 981 982 static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu) 983 { 984 struct dev_pm_opp *gpu_opp; 985 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index]; 986 987 gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true); 988 if (IS_ERR(gpu_opp)) 989 return; 990 991 dev_pm_opp_set_opp(&gpu->pdev->dev, gpu_opp); 992 dev_pm_opp_put(gpu_opp); 993 } 994 995 int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) 996 { 997 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 998 struct msm_gpu *gpu = &adreno_gpu->base; 999 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 1000 int status, ret; 1001 1002 if (WARN(!gmu->initialized, "The GMU is not set up yet\n")) 1003 return -EINVAL; 1004 1005 gmu->hung = false; 1006 1007 /* Notify AOSS about the ACD state (unimplemented for now => disable it) */ 1008 if (!IS_ERR(gmu->qmp)) { 1009 ret = qmp_send(gmu->qmp, "{class: gpu, res: acd, val: %d}", 1010 0 /* Hardcode ACD to be disabled for now */); 1011 if (ret) 1012 dev_err(gmu->dev, "failed to send GPU ACD state\n"); 1013 } 1014 1015 /* Turn on the resources */ 1016 pm_runtime_get_sync(gmu->dev); 1017 1018 /* 1019 * "enable" the GX power domain which won't actually do anything but it 1020 * will make sure that the refcounting is correct in case we need to 1021 * bring down the GX after a GMU failure 1022 */ 1023 if (!IS_ERR_OR_NULL(gmu->gxpd)) 1024 pm_runtime_get_sync(gmu->gxpd); 1025 1026 /* Use a known rate to bring up the GMU */ 1027 clk_set_rate(gmu->core_clk, 200000000); 1028 clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ? 1029 200000000 : 150000000); 1030 ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); 1031 if (ret) { 1032 pm_runtime_put(gmu->gxpd); 1033 pm_runtime_put(gmu->dev); 1034 return ret; 1035 } 1036 1037 /* Set the bus quota to a reasonable value for boot */ 1038 a6xx_gmu_set_initial_bw(gpu, gmu); 1039 1040 /* Enable the GMU interrupt */ 1041 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0); 1042 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK); 1043 enable_irq(gmu->gmu_irq); 1044 1045 /* Check to see if we are doing a cold or warm boot */ 1046 if (adreno_is_a7xx(adreno_gpu)) { 1047 status = a6xx_llc_read(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL) == 1 ? 1048 GMU_WARM_BOOT : GMU_COLD_BOOT; 1049 } else if (gmu->legacy) { 1050 status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ? 1051 GMU_WARM_BOOT : GMU_COLD_BOOT; 1052 } else { 1053 /* 1054 * Warm boot path does not work on newer A6xx GPUs 1055 * Presumably this is because icache/dcache regions must be restored 1056 */ 1057 status = GMU_COLD_BOOT; 1058 } 1059 1060 ret = a6xx_gmu_fw_start(gmu, status); 1061 if (ret) 1062 goto out; 1063 1064 ret = a6xx_hfi_start(gmu, status); 1065 if (ret) 1066 goto out; 1067 1068 /* 1069 * Turn on the GMU firmware fault interrupt after we know the boot 1070 * sequence is successful 1071 */ 1072 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0); 1073 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK); 1074 enable_irq(gmu->hfi_irq); 1075 1076 /* Set the GPU to the current freq */ 1077 a6xx_gmu_set_initial_freq(gpu, gmu); 1078 1079 out: 1080 /* On failure, shut down the GMU to leave it in a good state */ 1081 if (ret) { 1082 disable_irq(gmu->gmu_irq); 1083 a6xx_rpmh_stop(gmu); 1084 pm_runtime_put(gmu->gxpd); 1085 pm_runtime_put(gmu->dev); 1086 } 1087 1088 return ret; 1089 } 1090 1091 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) 1092 { 1093 u32 reg; 1094 1095 if (!gmu->initialized) 1096 return true; 1097 1098 reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS); 1099 1100 if (reg & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB) 1101 return false; 1102 1103 return true; 1104 } 1105 1106 /* Gracefully try to shut down the GMU and by extension the GPU */ 1107 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) 1108 { 1109 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 1110 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1111 u32 val; 1112 1113 /* 1114 * The GMU may still be in slumber unless the GPU started so check and 1115 * skip putting it back into slumber if so 1116 */ 1117 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE); 1118 1119 if (val != 0xf) { 1120 int ret = a6xx_gmu_wait_for_idle(gmu); 1121 1122 /* If the GMU isn't responding assume it is hung */ 1123 if (ret) { 1124 a6xx_gmu_force_off(gmu); 1125 return; 1126 } 1127 1128 a6xx_bus_clear_pending_transactions(adreno_gpu, a6xx_gpu->hung); 1129 1130 /* tell the GMU we want to slumber */ 1131 ret = a6xx_gmu_notify_slumber(gmu); 1132 if (ret) { 1133 a6xx_gmu_force_off(gmu); 1134 return; 1135 } 1136 1137 ret = gmu_poll_timeout(gmu, 1138 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val, 1139 !(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB), 1140 100, 10000); 1141 1142 /* 1143 * Let the user know we failed to slumber but don't worry too 1144 * much because we are powering down anyway 1145 */ 1146 1147 if (ret) 1148 DRM_DEV_ERROR(gmu->dev, 1149 "Unable to slumber GMU: status = 0%x/0%x\n", 1150 gmu_read(gmu, 1151 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS), 1152 gmu_read(gmu, 1153 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2)); 1154 } 1155 1156 /* Turn off HFI */ 1157 a6xx_hfi_stop(gmu); 1158 1159 /* Stop the interrupts and mask the hardware */ 1160 a6xx_gmu_irq_disable(gmu); 1161 1162 /* Tell RPMh to power off the GPU */ 1163 a6xx_rpmh_stop(gmu); 1164 } 1165 1166 1167 int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu) 1168 { 1169 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 1170 struct msm_gpu *gpu = &a6xx_gpu->base.base; 1171 1172 if (!pm_runtime_active(gmu->dev)) 1173 return 0; 1174 1175 /* 1176 * Force the GMU off if we detected a hang, otherwise try to shut it 1177 * down gracefully 1178 */ 1179 if (gmu->hung) 1180 a6xx_gmu_force_off(gmu); 1181 else 1182 a6xx_gmu_shutdown(gmu); 1183 1184 /* Remove the bus vote */ 1185 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); 1186 1187 /* 1188 * Make sure the GX domain is off before turning off the GMU (CX) 1189 * domain. Usually the GMU does this but only if the shutdown sequence 1190 * was successful 1191 */ 1192 if (!IS_ERR_OR_NULL(gmu->gxpd)) 1193 pm_runtime_put_sync(gmu->gxpd); 1194 1195 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); 1196 1197 pm_runtime_put_sync(gmu->dev); 1198 1199 return 0; 1200 } 1201 1202 static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu) 1203 { 1204 msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace); 1205 msm_gem_kernel_put(gmu->debug.obj, gmu->aspace); 1206 msm_gem_kernel_put(gmu->icache.obj, gmu->aspace); 1207 msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace); 1208 msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace); 1209 msm_gem_kernel_put(gmu->log.obj, gmu->aspace); 1210 1211 gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu); 1212 msm_gem_address_space_put(gmu->aspace); 1213 } 1214 1215 static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo, 1216 size_t size, u64 iova, const char *name) 1217 { 1218 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 1219 struct drm_device *dev = a6xx_gpu->base.base.dev; 1220 uint32_t flags = MSM_BO_WC; 1221 u64 range_start, range_end; 1222 int ret; 1223 1224 size = PAGE_ALIGN(size); 1225 if (!iova) { 1226 /* no fixed address - use GMU's uncached range */ 1227 range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */ 1228 range_end = 0x80000000; 1229 } else { 1230 /* range for fixed address */ 1231 range_start = iova; 1232 range_end = iova + size; 1233 /* use IOMMU_PRIV for icache/dcache */ 1234 flags |= MSM_BO_MAP_PRIV; 1235 } 1236 1237 bo->obj = msm_gem_new(dev, size, flags); 1238 if (IS_ERR(bo->obj)) 1239 return PTR_ERR(bo->obj); 1240 1241 ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova, 1242 range_start, range_end); 1243 if (ret) { 1244 drm_gem_object_put(bo->obj); 1245 return ret; 1246 } 1247 1248 bo->virt = msm_gem_get_vaddr(bo->obj); 1249 bo->size = size; 1250 1251 msm_gem_object_set_name(bo->obj, name); 1252 1253 return 0; 1254 } 1255 1256 static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu) 1257 { 1258 struct msm_mmu *mmu; 1259 1260 mmu = msm_iommu_new(gmu->dev, 0); 1261 if (!mmu) 1262 return -ENODEV; 1263 if (IS_ERR(mmu)) 1264 return PTR_ERR(mmu); 1265 1266 gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x80000000); 1267 if (IS_ERR(gmu->aspace)) 1268 return PTR_ERR(gmu->aspace); 1269 1270 return 0; 1271 } 1272 1273 /* Return the 'arc-level' for the given frequency */ 1274 static unsigned int a6xx_gmu_get_arc_level(struct device *dev, 1275 unsigned long freq) 1276 { 1277 struct dev_pm_opp *opp; 1278 unsigned int val; 1279 1280 if (!freq) 1281 return 0; 1282 1283 opp = dev_pm_opp_find_freq_exact(dev, freq, true); 1284 if (IS_ERR(opp)) 1285 return 0; 1286 1287 val = dev_pm_opp_get_level(opp); 1288 1289 dev_pm_opp_put(opp); 1290 1291 return val; 1292 } 1293 1294 static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes, 1295 unsigned long *freqs, int freqs_count, const char *id) 1296 { 1297 int i, j; 1298 const u16 *pri, *sec; 1299 size_t pri_count, sec_count; 1300 1301 pri = cmd_db_read_aux_data(id, &pri_count); 1302 if (IS_ERR(pri)) 1303 return PTR_ERR(pri); 1304 /* 1305 * The data comes back as an array of unsigned shorts so adjust the 1306 * count accordingly 1307 */ 1308 pri_count >>= 1; 1309 if (!pri_count) 1310 return -EINVAL; 1311 1312 /* 1313 * Some targets have a separate gfx mxc rail. So try to read that first and then fall back 1314 * to regular mx rail if it is missing 1315 */ 1316 sec = cmd_db_read_aux_data("gmxc.lvl", &sec_count); 1317 if (IS_ERR(sec) && sec != ERR_PTR(-EPROBE_DEFER)) 1318 sec = cmd_db_read_aux_data("mx.lvl", &sec_count); 1319 if (IS_ERR(sec)) 1320 return PTR_ERR(sec); 1321 1322 sec_count >>= 1; 1323 if (!sec_count) 1324 return -EINVAL; 1325 1326 /* Construct a vote for each frequency */ 1327 for (i = 0; i < freqs_count; i++) { 1328 u8 pindex = 0, sindex = 0; 1329 unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]); 1330 1331 /* Get the primary index that matches the arc level */ 1332 for (j = 0; j < pri_count; j++) { 1333 if (pri[j] >= level) { 1334 pindex = j; 1335 break; 1336 } 1337 } 1338 1339 if (j == pri_count) { 1340 DRM_DEV_ERROR(dev, 1341 "Level %u not found in the RPMh list\n", 1342 level); 1343 DRM_DEV_ERROR(dev, "Available levels:\n"); 1344 for (j = 0; j < pri_count; j++) 1345 DRM_DEV_ERROR(dev, " %u\n", pri[j]); 1346 1347 return -EINVAL; 1348 } 1349 1350 /* 1351 * Look for a level in in the secondary list that matches. If 1352 * nothing fits, use the maximum non zero vote 1353 */ 1354 1355 for (j = 0; j < sec_count; j++) { 1356 if (sec[j] >= level) { 1357 sindex = j; 1358 break; 1359 } else if (sec[j]) { 1360 sindex = j; 1361 } 1362 } 1363 1364 /* Construct the vote */ 1365 votes[i] = ((pri[pindex] & 0xffff) << 16) | 1366 (sindex << 8) | pindex; 1367 } 1368 1369 return 0; 1370 } 1371 1372 /* 1373 * The GMU votes with the RPMh for itself and on behalf of the GPU but we need 1374 * to construct the list of votes on the CPU and send it over. Query the RPMh 1375 * voltage levels and build the votes 1376 */ 1377 1378 static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu) 1379 { 1380 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 1381 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1382 struct msm_gpu *gpu = &adreno_gpu->base; 1383 int ret; 1384 1385 /* Build the GX votes */ 1386 ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes, 1387 gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl"); 1388 1389 /* Build the CX votes */ 1390 ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes, 1391 gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl"); 1392 1393 return ret; 1394 } 1395 1396 static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs, 1397 u32 size) 1398 { 1399 int count = dev_pm_opp_get_opp_count(dev); 1400 struct dev_pm_opp *opp; 1401 int i, index = 0; 1402 unsigned long freq = 1; 1403 1404 /* 1405 * The OPP table doesn't contain the "off" frequency level so we need to 1406 * add 1 to the table size to account for it 1407 */ 1408 1409 if (WARN(count + 1 > size, 1410 "The GMU frequency table is being truncated\n")) 1411 count = size - 1; 1412 1413 /* Set the "off" frequency */ 1414 freqs[index++] = 0; 1415 1416 for (i = 0; i < count; i++) { 1417 opp = dev_pm_opp_find_freq_ceil(dev, &freq); 1418 if (IS_ERR(opp)) 1419 break; 1420 1421 dev_pm_opp_put(opp); 1422 freqs[index++] = freq++; 1423 } 1424 1425 return index; 1426 } 1427 1428 static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu) 1429 { 1430 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 1431 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1432 struct msm_gpu *gpu = &adreno_gpu->base; 1433 1434 int ret = 0; 1435 1436 /* 1437 * The GMU handles its own frequency switching so build a list of 1438 * available frequencies to send during initialization 1439 */ 1440 ret = devm_pm_opp_of_add_table(gmu->dev); 1441 if (ret) { 1442 DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n"); 1443 return ret; 1444 } 1445 1446 gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev, 1447 gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs)); 1448 1449 /* 1450 * The GMU also handles GPU frequency switching so build a list 1451 * from the GPU OPP table 1452 */ 1453 gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev, 1454 gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs)); 1455 1456 gmu->current_perf_index = gmu->nr_gpu_freqs - 1; 1457 1458 /* Build the list of RPMh votes that we'll send to the GMU */ 1459 return a6xx_gmu_rpmh_votes_init(gmu); 1460 } 1461 1462 static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) 1463 { 1464 int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks); 1465 1466 if (ret < 1) 1467 return ret; 1468 1469 gmu->nr_clocks = ret; 1470 1471 gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks, 1472 gmu->nr_clocks, "gmu"); 1473 1474 gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks, 1475 gmu->nr_clocks, "hub"); 1476 1477 return 0; 1478 } 1479 1480 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, 1481 const char *name) 1482 { 1483 void __iomem *ret; 1484 struct resource *res = platform_get_resource_byname(pdev, 1485 IORESOURCE_MEM, name); 1486 1487 if (!res) { 1488 DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name); 1489 return ERR_PTR(-EINVAL); 1490 } 1491 1492 ret = ioremap(res->start, resource_size(res)); 1493 if (!ret) { 1494 DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name); 1495 return ERR_PTR(-EINVAL); 1496 } 1497 1498 return ret; 1499 } 1500 1501 static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, 1502 const char *name, irq_handler_t handler) 1503 { 1504 int irq, ret; 1505 1506 irq = platform_get_irq_byname(pdev, name); 1507 1508 ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH, name, gmu); 1509 if (ret) { 1510 DRM_DEV_ERROR(&pdev->dev, "Unable to get interrupt %s %d\n", 1511 name, ret); 1512 return ret; 1513 } 1514 1515 disable_irq(irq); 1516 1517 return irq; 1518 } 1519 1520 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) 1521 { 1522 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1523 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 1524 struct platform_device *pdev = to_platform_device(gmu->dev); 1525 1526 mutex_lock(&gmu->lock); 1527 if (!gmu->initialized) { 1528 mutex_unlock(&gmu->lock); 1529 return; 1530 } 1531 1532 gmu->initialized = false; 1533 1534 mutex_unlock(&gmu->lock); 1535 1536 pm_runtime_force_suspend(gmu->dev); 1537 1538 /* 1539 * Since cxpd is a virt device, the devlink with gmu-dev will be removed 1540 * automatically when we do detach 1541 */ 1542 dev_pm_domain_detach(gmu->cxpd, false); 1543 1544 if (!IS_ERR_OR_NULL(gmu->gxpd)) { 1545 pm_runtime_disable(gmu->gxpd); 1546 dev_pm_domain_detach(gmu->gxpd, false); 1547 } 1548 1549 if (!IS_ERR_OR_NULL(gmu->qmp)) 1550 qmp_put(gmu->qmp); 1551 1552 iounmap(gmu->mmio); 1553 if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) 1554 iounmap(gmu->rscc); 1555 gmu->mmio = NULL; 1556 gmu->rscc = NULL; 1557 1558 if (!adreno_has_gmu_wrapper(adreno_gpu)) { 1559 a6xx_gmu_memory_free(gmu); 1560 1561 free_irq(gmu->gmu_irq, gmu); 1562 free_irq(gmu->hfi_irq, gmu); 1563 } 1564 1565 /* Drop reference taken in of_find_device_by_node */ 1566 put_device(gmu->dev); 1567 } 1568 1569 static int cxpd_notifier_cb(struct notifier_block *nb, 1570 unsigned long action, void *data) 1571 { 1572 struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb); 1573 1574 if (action == GENPD_NOTIFY_OFF) 1575 complete_all(&gmu->pd_gate); 1576 1577 return 0; 1578 } 1579 1580 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) 1581 { 1582 struct platform_device *pdev = of_find_device_by_node(node); 1583 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 1584 int ret; 1585 1586 if (!pdev) 1587 return -ENODEV; 1588 1589 gmu->dev = &pdev->dev; 1590 1591 of_dma_configure(gmu->dev, node, true); 1592 1593 pm_runtime_enable(gmu->dev); 1594 1595 /* Mark legacy for manual SPTPRAC control */ 1596 gmu->legacy = true; 1597 1598 /* Map the GMU registers */ 1599 gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); 1600 if (IS_ERR(gmu->mmio)) { 1601 ret = PTR_ERR(gmu->mmio); 1602 goto err_mmio; 1603 } 1604 1605 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); 1606 if (IS_ERR(gmu->cxpd)) { 1607 ret = PTR_ERR(gmu->cxpd); 1608 goto err_mmio; 1609 } 1610 1611 if (!device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME)) { 1612 ret = -ENODEV; 1613 goto detach_cxpd; 1614 } 1615 1616 init_completion(&gmu->pd_gate); 1617 complete_all(&gmu->pd_gate); 1618 gmu->pd_nb.notifier_call = cxpd_notifier_cb; 1619 1620 /* Get a link to the GX power domain to reset the GPU */ 1621 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx"); 1622 if (IS_ERR(gmu->gxpd)) { 1623 ret = PTR_ERR(gmu->gxpd); 1624 goto err_mmio; 1625 } 1626 1627 gmu->initialized = true; 1628 1629 return 0; 1630 1631 detach_cxpd: 1632 dev_pm_domain_detach(gmu->cxpd, false); 1633 1634 err_mmio: 1635 iounmap(gmu->mmio); 1636 1637 /* Drop reference taken in of_find_device_by_node */ 1638 put_device(gmu->dev); 1639 1640 return ret; 1641 } 1642 1643 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) 1644 { 1645 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1646 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 1647 struct platform_device *pdev = of_find_device_by_node(node); 1648 struct device_link *link; 1649 int ret; 1650 1651 if (!pdev) 1652 return -ENODEV; 1653 1654 gmu->dev = &pdev->dev; 1655 1656 of_dma_configure(gmu->dev, node, true); 1657 1658 /* Fow now, don't do anything fancy until we get our feet under us */ 1659 gmu->idle_level = GMU_IDLE_STATE_ACTIVE; 1660 1661 pm_runtime_enable(gmu->dev); 1662 1663 /* Get the list of clocks */ 1664 ret = a6xx_gmu_clocks_probe(gmu); 1665 if (ret) 1666 goto err_put_device; 1667 1668 ret = a6xx_gmu_memory_probe(gmu); 1669 if (ret) 1670 goto err_put_device; 1671 1672 1673 /* A660 now requires handling "prealloc requests" in GMU firmware 1674 * For now just hardcode allocations based on the known firmware. 1675 * note: there is no indication that these correspond to "dummy" or 1676 * "debug" regions, but this "guess" allows reusing these BOs which 1677 * are otherwise unused by a660. 1678 */ 1679 gmu->dummy.size = SZ_4K; 1680 if (adreno_is_a660_family(adreno_gpu) || 1681 adreno_is_a7xx(adreno_gpu)) { 1682 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7, 1683 0x60400000, "debug"); 1684 if (ret) 1685 goto err_memory; 1686 1687 gmu->dummy.size = SZ_8K; 1688 } 1689 1690 /* Allocate memory for the GMU dummy page */ 1691 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, gmu->dummy.size, 1692 0x60000000, "dummy"); 1693 if (ret) 1694 goto err_memory; 1695 1696 /* Note that a650 family also includes a660 family: */ 1697 if (adreno_is_a650_family(adreno_gpu) || 1698 adreno_is_a7xx(adreno_gpu)) { 1699 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, 1700 SZ_16M - SZ_16K, 0x04000, "icache"); 1701 if (ret) 1702 goto err_memory; 1703 /* 1704 * NOTE: when porting legacy ("pre-650-family") GPUs you may be tempted to add a condition 1705 * to allocate icache/dcache here, as per downstream code flow, but it may not actually be 1706 * necessary. If you omit this step and you don't get random pagefaults, you are likely 1707 * good to go without this! 1708 */ 1709 } else if (adreno_is_a640_family(adreno_gpu)) { 1710 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache, 1711 SZ_256K - SZ_16K, 0x04000, "icache"); 1712 if (ret) 1713 goto err_memory; 1714 1715 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache, 1716 SZ_256K - SZ_16K, 0x44000, "dcache"); 1717 if (ret) 1718 goto err_memory; 1719 } else if (adreno_is_a630_family(adreno_gpu)) { 1720 /* HFI v1, has sptprac */ 1721 gmu->legacy = true; 1722 1723 /* Allocate memory for the GMU debug region */ 1724 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0, "debug"); 1725 if (ret) 1726 goto err_memory; 1727 } 1728 1729 /* Allocate memory for the GMU log region */ 1730 ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_16K, 0, "log"); 1731 if (ret) 1732 goto err_memory; 1733 1734 /* Allocate memory for for the HFI queues */ 1735 ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0, "hfi"); 1736 if (ret) 1737 goto err_memory; 1738 1739 /* Map the GMU registers */ 1740 gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); 1741 if (IS_ERR(gmu->mmio)) { 1742 ret = PTR_ERR(gmu->mmio); 1743 goto err_memory; 1744 } 1745 1746 if (adreno_is_a650_family(adreno_gpu) || 1747 adreno_is_a7xx(adreno_gpu)) { 1748 gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc"); 1749 if (IS_ERR(gmu->rscc)) { 1750 ret = -ENODEV; 1751 goto err_mmio; 1752 } 1753 } else { 1754 gmu->rscc = gmu->mmio + 0x23000; 1755 } 1756 1757 /* Get the HFI and GMU interrupts */ 1758 gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq); 1759 gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq); 1760 1761 if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) { 1762 ret = -ENODEV; 1763 goto err_mmio; 1764 } 1765 1766 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); 1767 if (IS_ERR(gmu->cxpd)) { 1768 ret = PTR_ERR(gmu->cxpd); 1769 goto err_mmio; 1770 } 1771 1772 link = device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME); 1773 if (!link) { 1774 ret = -ENODEV; 1775 goto detach_cxpd; 1776 } 1777 1778 gmu->qmp = qmp_get(gmu->dev); 1779 if (IS_ERR(gmu->qmp) && adreno_is_a7xx(adreno_gpu)) { 1780 ret = PTR_ERR(gmu->qmp); 1781 goto remove_device_link; 1782 } 1783 1784 init_completion(&gmu->pd_gate); 1785 complete_all(&gmu->pd_gate); 1786 gmu->pd_nb.notifier_call = cxpd_notifier_cb; 1787 1788 /* 1789 * Get a link to the GX power domain to reset the GPU in case of GMU 1790 * crash 1791 */ 1792 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx"); 1793 1794 /* Get the power levels for the GMU and GPU */ 1795 a6xx_gmu_pwrlevels_probe(gmu); 1796 1797 /* Set up the HFI queues */ 1798 a6xx_hfi_init(gmu); 1799 1800 /* Initialize RPMh */ 1801 a6xx_gmu_rpmh_init(gmu); 1802 1803 gmu->initialized = true; 1804 1805 return 0; 1806 1807 remove_device_link: 1808 device_link_del(link); 1809 1810 detach_cxpd: 1811 dev_pm_domain_detach(gmu->cxpd, false); 1812 1813 err_mmio: 1814 iounmap(gmu->mmio); 1815 if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) 1816 iounmap(gmu->rscc); 1817 free_irq(gmu->gmu_irq, gmu); 1818 free_irq(gmu->hfi_irq, gmu); 1819 1820 err_memory: 1821 a6xx_gmu_memory_free(gmu); 1822 err_put_device: 1823 /* Drop reference taken in of_find_device_by_node */ 1824 put_device(gmu->dev); 1825 1826 return ret; 1827 } 1828