1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
3
4 #include <linux/bitfield.h>
5 #include <linux/clk.h>
6 #include <linux/firmware/qcom/qcom_scm.h>
7 #include <linux/interconnect.h>
8 #include <linux/of_platform.h>
9 #include <linux/platform_device.h>
10 #include <linux/pm_domain.h>
11 #include <linux/pm_opp.h>
12 #include <soc/qcom/cmd-db.h>
13 #include <soc/qcom/tcs.h>
14 #include <drm/drm_gem.h>
15
16 #include "a6xx_gpu.h"
17 #include "a6xx_gmu.xml.h"
18 #include "msm_gem.h"
19 #include "msm_gpu_trace.h"
20 #include "msm_mmu.h"
21
a6xx_gmu_fault(struct a6xx_gmu * gmu)22 static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
23 {
24 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
25 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
26 struct msm_gpu *gpu = &adreno_gpu->base;
27
28 /* FIXME: add a banner here */
29 gmu->hung = true;
30
31 /* Turn off the hangcheck timer while we are resetting */
32 timer_delete(&gpu->hangcheck_timer);
33
34 /* Queue the GPU handler because we need to treat this as a recovery */
35 kthread_queue_work(gpu->worker, &gpu->recover_work);
36 }
37
a6xx_gmu_irq(int irq,void * data)38 static irqreturn_t a6xx_gmu_irq(int irq, void *data)
39 {
40 struct a6xx_gmu *gmu = data;
41 u32 status;
42
43 status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS);
44 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
45
46 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
47 dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
48
49 a6xx_gmu_fault(gmu);
50 }
51
52 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
53 dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n");
54
55 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
56 dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n",
57 gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS));
58
59 return IRQ_HANDLED;
60 }
61
a6xx_hfi_irq(int irq,void * data)62 static irqreturn_t a6xx_hfi_irq(int irq, void *data)
63 {
64 struct a6xx_gmu *gmu = data;
65 u32 status;
66
67 status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
68 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
69
70 if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
71 dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
72
73 a6xx_gmu_fault(gmu);
74 }
75
76 return IRQ_HANDLED;
77 }
78
a6xx_gmu_sptprac_is_on(struct a6xx_gmu * gmu)79 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu)
80 {
81 u32 val;
82
83 /* This can be called from gpu state code so make sure GMU is valid */
84 if (!gmu->initialized)
85 return false;
86
87 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
88
89 return !(val &
90 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF |
91 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF));
92 }
93
94 /* Check to see if the GX rail is still powered */
a6xx_gmu_gx_is_on(struct adreno_gpu * adreno_gpu)95 bool a6xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu)
96 {
97 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
98 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
99 u32 val;
100
101 /* This can be called from gpu state code so make sure GMU is valid */
102 if (!gmu->initialized)
103 return false;
104
105 /* If GMU is absent, then GX power domain is ON as long as GPU is in active state */
106 if (adreno_has_gmu_wrapper(adreno_gpu))
107 return true;
108
109 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
110
111 if (adreno_is_a7xx(adreno_gpu))
112 return !(val &
113 (A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
114 A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
115
116 return !(val &
117 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
118 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
119 }
120
a7xx_gmu_gx_is_on(struct adreno_gpu * adreno_gpu)121 bool a7xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu)
122 {
123 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
124 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
125 u32 val;
126
127 /* This can be called from gpu state code so make sure GMU is valid */
128 if (!gmu->initialized)
129 return false;
130
131 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
132
133 return !(val &
134 (A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
135 A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
136 }
137
a8xx_gmu_gx_is_on(struct adreno_gpu * adreno_gpu)138 bool a8xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu)
139 {
140 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
141 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
142 u32 val;
143
144 /* This can be called from gpu state code so make sure GMU is valid */
145 if (!gmu->initialized)
146 return false;
147
148 val = gmu_read(gmu, REG_A8XX_GMU_PWR_CLK_STATUS);
149
150 return !(val &
151 (A8XX_GMU_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
152 A8XX_GMU_PWR_CLK_STATUS_GX_HM_CLK_OFF));
153 }
154
a6xx_gmu_set_freq(struct msm_gpu * gpu,struct dev_pm_opp * opp,bool suspended)155 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
156 bool suspended)
157 {
158 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
159 const struct a6xx_info *info = adreno_gpu->info->a6xx;
160 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
161 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
162 u32 perf_index;
163 u32 bw_index = 0;
164 unsigned long gpu_freq;
165 int ret = 0;
166
167 gpu_freq = dev_pm_opp_get_freq(opp);
168
169 if (gpu_freq == gmu->freq)
170 return;
171
172 for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++)
173 if (gpu_freq == gmu->gpu_freqs[perf_index])
174 break;
175
176 /* If enabled, find the corresponding DDR bandwidth index */
177 if (info->bcms && gmu->nr_gpu_bws > 1) {
178 unsigned int bw = dev_pm_opp_get_bw(opp, true, 0);
179
180 for (bw_index = 0; bw_index < gmu->nr_gpu_bws - 1; bw_index++) {
181 if (bw == gmu->gpu_bw_table[bw_index])
182 break;
183 }
184
185 /* Vote AB as a fraction of the max bandwidth, starting from A750 */
186 if (bw && adreno_is_a750_family(adreno_gpu)) {
187 u64 tmp;
188
189 /* For now, vote for 25% of the bandwidth */
190 tmp = bw * 25;
191 do_div(tmp, 100);
192
193 /*
194 * The AB vote consists of a 16 bit wide quantized level
195 * against the maximum supported bandwidth.
196 * Quantization can be calculated as below:
197 * vote = (bandwidth * 2^16) / max bandwidth
198 */
199 tmp *= MAX_AB_VOTE;
200 do_div(tmp, gmu->gpu_bw_table[gmu->nr_gpu_bws - 1]);
201
202 bw_index |= AB_VOTE(clamp(tmp, 1, MAX_AB_VOTE));
203 bw_index |= AB_VOTE_ENABLE;
204 }
205 }
206
207 gmu->current_perf_index = perf_index;
208 gmu->freq = gmu->gpu_freqs[perf_index];
209
210 trace_msm_gmu_freq_change(gmu->freq, perf_index);
211
212 /*
213 * This can get called from devfreq while the hardware is idle. Don't
214 * bring up the power if it isn't already active. All we're doing here
215 * is updating the frequency so that when we come back online we're at
216 * the right rate.
217 */
218 if (suspended)
219 return;
220
221 if (!gmu->legacy) {
222 a6xx_hfi_set_freq(gmu, perf_index, bw_index);
223 /* With Bandwidth voting, we now vote for all resources, so skip OPP set */
224 if (!bw_index)
225 dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
226 return;
227 }
228
229 gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
230
231 gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
232 ((3 & 0xf) << 28) | perf_index);
233
234 /*
235 * Send an invalid index as a vote for the bus bandwidth and let the
236 * firmware decide on the right vote
237 */
238 gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff);
239
240 /* Set and clear the OOB for DCVS to trigger the GMU */
241 a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
242 a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
243
244 ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
245 if (ret)
246 dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
247
248 dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
249 }
250
a6xx_gmu_get_freq(struct msm_gpu * gpu)251 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
252 {
253 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
254 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
255 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
256
257 return gmu->freq;
258 }
259
a6xx_gmu_check_idle_level(struct a6xx_gmu * gmu)260 static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
261 {
262 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
263 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
264 int local = gmu->idle_level;
265 u32 val;
266
267 /* SPTP and IFPC both report as IFPC */
268 if (gmu->idle_level == GMU_IDLE_STATE_SPTP)
269 local = GMU_IDLE_STATE_IFPC;
270
271 if (adreno_is_a8xx(adreno_gpu))
272 val = gmu_read(gmu, REG_A8XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
273 else
274 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
275
276 if (val == local) {
277 if (gmu->idle_level != GMU_IDLE_STATE_IFPC ||
278 !adreno_gpu->funcs->gx_is_on(adreno_gpu))
279 return true;
280 }
281
282 return false;
283 }
284
285 /* Wait for the GMU to get to its most idle state */
a6xx_gmu_wait_for_idle(struct a6xx_gmu * gmu)286 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu)
287 {
288 return spin_until(a6xx_gmu_check_idle_level(gmu));
289 }
290
a6xx_gmu_start(struct a6xx_gmu * gmu)291 static int a6xx_gmu_start(struct a6xx_gmu *gmu)
292 {
293 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
294 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
295 u32 mask, reset_val, val;
296 int ret;
297
298 val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8);
299 if (val <= 0x20010004) {
300 mask = 0xffffffff;
301 reset_val = 0xbabeface;
302 } else {
303 mask = 0x1ff;
304 reset_val = 0x100;
305 }
306
307 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
308
309 /* Set the log wptr index
310 * note: downstream saves the value in poweroff and restores it here
311 */
312 if (adreno_is_a8xx(adreno_gpu))
313 gmu_write(gmu, REG_A8XX_GMU_GENERAL_9, 0);
314 else if (adreno_is_a7xx(adreno_gpu))
315 gmu_write(gmu, REG_A7XX_GMU_GENERAL_9, 0);
316 else
317 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0);
318
319
320 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
321
322 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
323 (val & mask) == reset_val, 100, 100000);
324
325 if (ret)
326 DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n");
327
328 set_bit(GMU_STATUS_FW_START, &gmu->status);
329
330 return ret;
331 }
332
a6xx_gmu_hfi_start(struct a6xx_gmu * gmu)333 static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu)
334 {
335 u32 val;
336 int ret;
337
338 gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
339
340 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
341 val & 1, 100, 10000);
342 if (ret)
343 DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n");
344
345 return ret;
346 }
347
348 struct a6xx_gmu_oob_bits {
349 int set, ack, set_new, ack_new, clear, clear_new;
350 const char *name;
351 };
352
353 /* These are the interrupt / ack bits for each OOB request that are set
354 * in a6xx_gmu_set_oob and a6xx_clear_oob
355 */
356 static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = {
357 [GMU_OOB_GPU_SET] = {
358 .name = "GPU_SET",
359 .set = 16,
360 .ack = 24,
361 .set_new = 30,
362 .ack_new = 31,
363 .clear = 24,
364 .clear_new = 31,
365 },
366
367 [GMU_OOB_PERFCOUNTER_SET] = {
368 .name = "PERFCOUNTER",
369 .set = 17,
370 .ack = 25,
371 .set_new = 28,
372 .ack_new = 30,
373 .clear = 25,
374 .clear_new = 29,
375 },
376
377 [GMU_OOB_BOOT_SLUMBER] = {
378 .name = "BOOT_SLUMBER",
379 .set = 22,
380 .ack = 30,
381 .clear = 30,
382 },
383
384 [GMU_OOB_DCVS_SET] = {
385 .name = "GPU_DCVS",
386 .set = 23,
387 .ack = 31,
388 .clear = 31,
389 },
390 };
391
392 /* Trigger a OOB (out of band) request to the GMU */
a6xx_gmu_set_oob(struct a6xx_gmu * gmu,enum a6xx_gmu_oob_state state)393 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
394 {
395 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
396 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
397 int ret;
398 u32 val;
399 int request, ack;
400
401 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
402
403 /* Skip OOB calls since RGMU is not enabled */
404 if (adreno_has_rgmu(adreno_gpu))
405 return 0;
406
407 if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
408 return -EINVAL;
409
410 if (gmu->legacy) {
411 request = a6xx_gmu_oob_bits[state].set;
412 ack = a6xx_gmu_oob_bits[state].ack;
413 } else {
414 request = a6xx_gmu_oob_bits[state].set_new;
415 ack = a6xx_gmu_oob_bits[state].ack_new;
416 if (!request || !ack) {
417 DRM_DEV_ERROR(gmu->dev,
418 "Invalid non-legacy GMU request %s\n",
419 a6xx_gmu_oob_bits[state].name);
420 return -EINVAL;
421 }
422 }
423
424 /* Trigger the equested OOB operation */
425 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request);
426
427 do {
428 /* Wait for the acknowledge interrupt */
429 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
430 val & (1 << ack), 100, 10000);
431
432 if (!ret)
433 break;
434
435 if (completion_done(&a6xx_gpu->base.fault_coredump_done))
436 break;
437
438 /* We may timeout because the GMU is temporarily wedged from
439 * pending faults from the GPU and we are taking a devcoredump.
440 * Wait until the MMU is resumed and try again.
441 */
442 wait_for_completion(&a6xx_gpu->base.fault_coredump_done);
443 } while (true);
444
445 if (ret)
446 DRM_DEV_ERROR(gmu->dev,
447 "Timeout waiting for GMU OOB set %s: 0x%x\n",
448 a6xx_gmu_oob_bits[state].name,
449 gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
450
451 /* Clear the acknowledge interrupt */
452 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack);
453
454 return ret;
455 }
456
457 /* Clear a pending OOB state in the GMU */
a6xx_gmu_clear_oob(struct a6xx_gmu * gmu,enum a6xx_gmu_oob_state state)458 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
459 {
460 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
461 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
462 int bit;
463
464 WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
465
466 /* Skip OOB calls since RGMU is not enabled */
467 if (adreno_has_rgmu(adreno_gpu))
468 return;
469
470 if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
471 return;
472
473 if (gmu->legacy)
474 bit = a6xx_gmu_oob_bits[state].clear;
475 else
476 bit = a6xx_gmu_oob_bits[state].clear_new;
477
478 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit);
479 }
480
481 /* Enable CPU control of SPTP power power collapse */
a6xx_sptprac_enable(struct a6xx_gmu * gmu)482 int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
483 {
484 int ret;
485 u32 val;
486
487 WARN_ON(!gmu->legacy);
488
489 /* Nothing to do if GMU does the power management */
490 if (gmu->idle_level > GMU_IDLE_STATE_ACTIVE)
491 return 0;
492
493 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
494
495 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
496 (val & 0x38) == 0x28, 1, 100);
497
498 if (ret) {
499 DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
500 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
501 }
502
503 return 0;
504 }
505
506 /* Disable CPU control of SPTP power power collapse */
a6xx_sptprac_disable(struct a6xx_gmu * gmu)507 void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
508 {
509 u32 val;
510 int ret;
511
512 if (!gmu->legacy)
513 return;
514
515 /* Make sure retention is on */
516 gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
517
518 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001);
519
520 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
521 (val & 0x04), 100, 10000);
522
523 if (ret)
524 DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
525 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
526 }
527
528 /* Let the GMU know we are starting a boot sequence */
a6xx_gmu_gfx_rail_on(struct a6xx_gmu * gmu)529 static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu)
530 {
531 u32 vote;
532
533 /* Let the GMU know we are getting ready for boot */
534 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0);
535
536 /* Choose the "default" power level as the highest available */
537 vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1];
538
539 gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff);
540 gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff);
541
542 /* Let the GMU know the boot sequence has started */
543 return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
544 }
545
a6xx_gemnoc_workaround(struct a6xx_gmu * gmu)546 static void a6xx_gemnoc_workaround(struct a6xx_gmu *gmu)
547 {
548 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
549 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
550
551 /*
552 * GEMNoC can power collapse whilst the GPU is being powered down, resulting
553 * in the power down sequence not being fully executed. That in turn can
554 * prevent CX_GDSC from collapsing. Assert Qactive to avoid this.
555 */
556 if (adreno_is_a8xx(adreno_gpu))
557 gmu_write(gmu, REG_A8XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, BIT(0));
558 else if (adreno_is_a7xx(adreno_gpu) || (adreno_is_a621(adreno_gpu) ||
559 adreno_is_7c3(adreno_gpu)))
560 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, BIT(0));
561 }
562
563 /* Let the GMU know that we are about to go into slumber */
a6xx_gmu_notify_slumber(struct a6xx_gmu * gmu)564 static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
565 {
566 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
567 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
568 int ret;
569
570 /* Disable the power counter so the GMU isn't busy */
571 if (adreno_is_a8xx(adreno_gpu))
572 gmu_write(gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
573 else
574 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
575
576 /* Disable SPTP_PC if the CPU is responsible for it */
577 if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
578 a6xx_sptprac_disable(gmu);
579
580 if (!gmu->legacy) {
581 ret = a6xx_hfi_send_prep_slumber(gmu);
582 goto out;
583 }
584
585 /* Tell the GMU to get ready to slumber */
586 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
587
588 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
589 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
590
591 if (!ret) {
592 /* Check to see if the GMU really did slumber */
593 if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
594 != 0x0f) {
595 DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n");
596 ret = -ETIMEDOUT;
597 }
598 }
599
600 out:
601 /* Put fence into allow mode */
602 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
603 a6xx_gemnoc_workaround(gmu);
604 return ret;
605 }
606
a6xx_rpmh_start(struct a6xx_gmu * gmu)607 static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
608 {
609 int ret;
610 u32 val;
611
612 if (!test_and_clear_bit(GMU_STATUS_PDC_SLEEP, &gmu->status))
613 return 0;
614
615 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, BIT(1));
616
617 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
618 val & (1 << 1), 100, 10000);
619 if (ret) {
620 DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n");
621 return ret;
622 }
623
624 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
625 !val, 100, 10000);
626
627 if (ret) {
628 DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
629 return ret;
630 }
631
632 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
633
634 return 0;
635 }
636
a6xx_rpmh_stop(struct a6xx_gmu * gmu)637 static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
638 {
639 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
640 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
641 u32 bitmask = BIT(16);
642 int ret;
643 u32 val;
644
645 if (!test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status))
646 return;
647
648 if (adreno_is_a840(adreno_gpu))
649 bitmask = BIT(30);
650
651 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
652
653 ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
654 val, val & bitmask, 100, 10000);
655 if (ret)
656 DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n");
657
658 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
659
660 set_bit(GMU_STATUS_PDC_SLEEP, &gmu->status);
661 }
662
pdc_write(void __iomem * ptr,u32 offset,u32 value)663 static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value)
664 {
665 writel(value, ptr + (offset << 2));
666 }
667
a6xx_gmu_rpmh_init(struct a6xx_gmu * gmu)668 static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
669 {
670 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
671 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
672 struct platform_device *pdev = to_platform_device(gmu->dev);
673 u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0;
674 void __iomem *seqptr = NULL;
675 uint32_t pdc_address_offset;
676 void __iomem *pdcptr;
677 bool pdc_in_aop = false;
678
679 /* On A8x and above, RPMH/PDC configurations are entirely configured in AOP */
680 if (adreno_is_a8xx(adreno_gpu))
681 return;
682
683 pdcptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc");
684 if (IS_ERR(pdcptr))
685 return;
686
687 if (adreno_is_a650_family(adreno_gpu) ||
688 adreno_is_a7xx(adreno_gpu))
689 pdc_in_aop = true;
690 else if (adreno_is_a618(adreno_gpu) || adreno_is_a640_family(adreno_gpu))
691 pdc_address_offset = 0x30090;
692 else if (adreno_is_a619(adreno_gpu))
693 pdc_address_offset = 0x300a0;
694 else
695 pdc_address_offset = 0x30080;
696
697 if (!pdc_in_aop) {
698 seqptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc_seq");
699 if (IS_ERR(seqptr))
700 return;
701 }
702
703 /* Disable SDE clock gating */
704 gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
705
706 /* Setup RSC PDC handshake for sleep and wakeup */
707 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
708 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
709 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
710 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
711 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
712 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4,
713 (adreno_is_a740_family(adreno_gpu) ||
714 adreno_is_a722(adreno_gpu)) ? 0x80000021 : 0x80000000);
715 gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
716 gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
717 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
718 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
719 gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
720
721 /* The second spin of A7xx GPUs messed with some register offsets.. */
722 if (adreno_is_a740_family(adreno_gpu) || adreno_is_a722(adreno_gpu))
723 seqmem0_drv0_reg = REG_A7XX_RSCC_SEQ_MEM_0_DRV0_A740;
724
725 /* Load RSC sequencer uCode for sleep and wakeup */
726 if (adreno_is_a650_family(adreno_gpu) ||
727 adreno_is_a7xx(adreno_gpu)) {
728 gmu_write_rscc(gmu, seqmem0_drv0_reg, 0xeaaae5a0);
729 gmu_write_rscc(gmu, seqmem0_drv0_reg + 1, 0xe1a1ebab);
730 gmu_write_rscc(gmu, seqmem0_drv0_reg + 2, 0xa2e0a581);
731 gmu_write_rscc(gmu, seqmem0_drv0_reg + 3, 0xecac82e2);
732 gmu_write_rscc(gmu, seqmem0_drv0_reg + 4, 0x0020edad);
733 } else {
734 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
735 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
736 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
737 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
738 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
739 }
740
741 if (pdc_in_aop)
742 goto setup_pdc;
743
744 /* Load PDC sequencer uCode for power up and power down sequence */
745 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
746 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
747 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
748 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
749 pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
750
751 /* Set TCS commands used by PDC sequence for low power modes */
752 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
753 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
754 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
755 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
756 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
757 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
758 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
759 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
760 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
761
762 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
763 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset);
764 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
765
766 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
767 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
768 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
769 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
770 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
771 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
772
773 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
774 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
775 if (adreno_is_a618(adreno_gpu) || adreno_is_a619(adreno_gpu) ||
776 adreno_is_a650_family(adreno_gpu))
777 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2);
778 else
779 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
780 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
781 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset);
782 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
783
784 /* Setup GPU PDC */
785 setup_pdc:
786 pdc_write(pdcptr, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
787 pdc_write(pdcptr, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
788
789 /* ensure no writes happen before the uCode is fully written */
790 wmb();
791 }
792
793 /*
794 * The lowest 16 bits of this value are the number of XO clock cycles for main
795 * hysteresis which is set at 0x1680 cycles (300 us). The higher 16 bits are
796 * for the shorter hysteresis that happens after main - this is 0xa (.5 us)
797 */
798
799 #define GMU_PWR_COL_HYST 0x000a1680
800
801 /* Set up the idle state for the GMU */
a6xx_gmu_power_config(struct a6xx_gmu * gmu)802 static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
803 {
804 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
805 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
806
807 /* Disable GMU WB/RB buffer */
808 gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
809 gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1);
810 gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1);
811
812 /* A7xx knows better by default! */
813 if (adreno_is_a7xx(adreno_gpu) || adreno_is_a8xx(adreno_gpu))
814 return;
815
816 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
817
818 switch (gmu->idle_level) {
819 case GMU_IDLE_STATE_IFPC:
820 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
821 GMU_PWR_COL_HYST);
822 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
823 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
824 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
825 fallthrough;
826 case GMU_IDLE_STATE_SPTP:
827 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
828 GMU_PWR_COL_HYST);
829 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
830 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
831 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE);
832 }
833
834 /* Enable RPMh GPU client */
835 gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0,
836 A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE |
837 A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE |
838 A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE |
839 A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE |
840 A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE |
841 A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
842 }
843
844 struct block_header {
845 u32 addr;
846 u32 size;
847 u32 type;
848 u32 value;
849 u32 data[];
850 };
851
fw_block_mem(struct a6xx_gmu_bo * bo,const struct block_header * blk)852 static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk)
853 {
854 if (!in_range(blk->addr, bo->iova, bo->size))
855 return false;
856
857 memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size);
858 return true;
859 }
860
861 #define NEXT_BLK(blk) \
862 ((const struct block_header *)((const char *)(blk) + sizeof(*(blk)) + (blk)->size))
863
a6xx_gmu_fw_load(struct a6xx_gmu * gmu)864 static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
865 {
866 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
867 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
868 const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU];
869 const struct block_header *blk;
870 u32 reg_offset;
871 u32 ver;
872
873 u32 itcm_base = 0x00000000;
874 u32 dtcm_base = 0x00040000;
875
876 if (adreno_is_a650_family(adreno_gpu) ||
877 adreno_is_a7xx(adreno_gpu) ||
878 adreno_is_a8xx(adreno_gpu))
879 dtcm_base = 0x10004000;
880
881 if (gmu->legacy) {
882 /* Sanity check the size of the firmware that was loaded */
883 if (fw_image->size > 0x8000) {
884 DRM_DEV_ERROR(gmu->dev,
885 "GMU firmware is bigger than the available region\n");
886 return -EINVAL;
887 }
888
889 gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START,
890 (u32*) fw_image->data, fw_image->size);
891 return 0;
892 }
893
894
895 for (blk = (const struct block_header *) fw_image->data;
896 (const u8*) blk < fw_image->data + fw_image->size;
897 blk = NEXT_BLK(blk)) {
898 if (blk->size == 0)
899 continue;
900
901 if (in_range(blk->addr, itcm_base, SZ_16K)) {
902 reg_offset = (blk->addr - itcm_base) >> 2;
903 gmu_write_bulk(gmu,
904 REG_A6XX_GMU_CM3_ITCM_START + reg_offset,
905 blk->data, blk->size);
906 } else if (in_range(blk->addr, dtcm_base, SZ_16K)) {
907 reg_offset = (blk->addr - dtcm_base) >> 2;
908 gmu_write_bulk(gmu,
909 REG_A6XX_GMU_CM3_DTCM_START + reg_offset,
910 blk->data, blk->size);
911 } else if (!fw_block_mem(&gmu->icache, blk) &&
912 !fw_block_mem(&gmu->dcache, blk) &&
913 !fw_block_mem(&gmu->dummy, blk)) {
914 DRM_DEV_ERROR(gmu->dev,
915 "failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n",
916 blk->addr, blk->size, blk->data[0]);
917 }
918 }
919
920 ver = gmu_read(gmu, REG_A6XX_GMU_CORE_FW_VERSION);
921 DRM_INFO_ONCE("Loaded GMU firmware v%u.%u.%u\n",
922 FIELD_GET(A6XX_GMU_CORE_FW_VERSION_MAJOR__MASK, ver),
923 FIELD_GET(A6XX_GMU_CORE_FW_VERSION_MINOR__MASK, ver),
924 FIELD_GET(A6XX_GMU_CORE_FW_VERSION_STEP__MASK, ver));
925
926 return 0;
927 }
928
a6xx_gmu_fw_start(struct a6xx_gmu * gmu,unsigned int state)929 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
930 {
931 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
932 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
933 struct msm_gpu *gpu = &adreno_gpu->base;
934 const struct a6xx_info *a6xx_info = adreno_gpu->info->a6xx;
935 const struct adreno_reglist *gbif_cx = a6xx_info->gbif_cx;
936 u32 fence_range_lower, fence_range_upper;
937 u32 chipid = 0;
938 int ret;
939
940 /* Vote veto for FAL10 */
941 if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) {
942 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1);
943 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
944 } else if (adreno_is_a8xx(adreno_gpu)) {
945 gmu_write(gmu, REG_A8XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1);
946 gmu_write(gmu, REG_A8XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
947 }
948
949 /* Turn on TCM (Tightly Coupled Memory) retention */
950 if (adreno_is_a7xx(adreno_gpu))
951 a6xx_cx_misc_write(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL, 1);
952 else if (!adreno_is_a8xx(adreno_gpu))
953 gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
954
955 ret = a6xx_rpmh_start(gmu);
956 if (ret)
957 return ret;
958
959 if (state == GMU_COLD_BOOT) {
960 if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU],
961 "GMU firmware is not loaded\n"))
962 return -ENOENT;
963
964 ret = a6xx_gmu_fw_load(gmu);
965 if (ret)
966 return ret;
967 }
968
969 /* Clear init result to make sure we are getting a fresh value */
970 gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
971 gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
972
973 /* Write the iova of the HFI table */
974 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova);
975 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
976
977 if (adreno_is_a8xx(adreno_gpu)) {
978 fence_range_upper = 0x32;
979 fence_range_lower = 0x8c0;
980 } else if (adreno_is_a7xx(adreno_gpu)) {
981 fence_range_upper = 0x32;
982 fence_range_lower = 0x8a0;
983 } else {
984 fence_range_upper = 0xa;
985 fence_range_lower = 0xa0;
986 }
987
988 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
989 BIT(31) |
990 FIELD_PREP(GENMASK(30, 18), fence_range_upper) |
991 FIELD_PREP(GENMASK(17, 0), fence_range_lower));
992
993 /*
994 * Snapshots toggle the NMI bit which will result in a jump to the NMI
995 * handler instead of __main. Set the M3 config value to avoid that.
996 */
997 gmu_write(gmu, REG_A6XX_GMU_CM3_CFG, 0x4052);
998
999 if (a6xx_info->gmu_chipid) {
1000 chipid = a6xx_info->gmu_chipid;
1001 } else {
1002 /*
1003 * Note that the GMU has a slightly different layout for
1004 * chip_id, for whatever reason, so a bit of massaging
1005 * is needed. The upper 16b are the same, but minor and
1006 * patchid are packed in four bits each with the lower
1007 * 8b unused:
1008 */
1009 chipid = adreno_gpu->chip_id & 0xffff0000;
1010 chipid |= (adreno_gpu->chip_id << 4) & 0xf000; /* minor */
1011 chipid |= (adreno_gpu->chip_id << 8) & 0x0f00; /* patchid */
1012 }
1013
1014 if (adreno_is_a8xx(adreno_gpu)) {
1015 gmu_write(gmu, REG_A8XX_GMU_GENERAL_10, chipid);
1016 gmu_write(gmu, REG_A8XX_GMU_GENERAL_8,
1017 (gmu->log.iova & GENMASK(31, 12)) |
1018 ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0)));
1019 } else if (adreno_is_a7xx(adreno_gpu)) {
1020 gmu_write(gmu, REG_A7XX_GMU_GENERAL_10, chipid);
1021 gmu_write(gmu, REG_A7XX_GMU_GENERAL_8,
1022 (gmu->log.iova & GENMASK(31, 12)) |
1023 ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0)));
1024 } else {
1025 gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
1026
1027 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
1028 gmu->log.iova | (gmu->log.size / SZ_4K - 1));
1029 }
1030
1031 /* For A7x and newer, do the CX GBIF configurations before GMU wake up */
1032 for (int i = 0; (gbif_cx && gbif_cx[i].offset); i++)
1033 gpu_write(gpu, gbif_cx[i].offset, gbif_cx[i].value);
1034
1035 if (adreno_is_a8xx(adreno_gpu)) {
1036 gpu_write(gpu, REG_A6XX_GBIF_CX_CONFIG, 0x20023000);
1037 gmu_write(gmu, REG_A6XX_GMU_MRC_GBIF_QOS_CTRL, 0x33);
1038 } else if (adreno_is_a722(adreno_gpu))
1039 gpu_rmw(gpu, REG_A6XX_GBIF_CX_CONFIG, GENMASK(31, 29),
1040 FIELD_PREP(GENMASK(31, 29), 2));
1041
1042 /* Set up the lowest idle level on the GMU */
1043 a6xx_gmu_power_config(gmu);
1044
1045 ret = a6xx_gmu_start(gmu);
1046 if (ret)
1047 return ret;
1048
1049 if (gmu->legacy) {
1050 ret = a6xx_gmu_gfx_rail_on(gmu);
1051 if (ret)
1052 return ret;
1053
1054 ret = a6xx_sptprac_enable(gmu);
1055 if (ret)
1056 return ret;
1057 }
1058
1059 ret = a6xx_gmu_hfi_start(gmu);
1060 if (ret)
1061 return ret;
1062
1063 /* FIXME: Do we need this wmb() here? */
1064 wmb();
1065
1066 return 0;
1067 }
1068
1069 #define A6XX_HFI_IRQ_MASK \
1070 (A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
1071
1072 #define A6XX_GMU_IRQ_MASK \
1073 (A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \
1074 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
1075 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
1076
a6xx_gmu_irq_disable(struct a6xx_gmu * gmu)1077 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
1078 {
1079 disable_irq(gmu->gmu_irq);
1080 disable_irq(gmu->hfi_irq);
1081
1082 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0);
1083 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
1084 }
1085
a6xx_gmu_rpmh_off(struct a6xx_gmu * gmu)1086 static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
1087 {
1088 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1089 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1090 u32 val, seqmem_off = 0;
1091
1092 /* The second spin of A7xx GPUs messed with some register offsets.. */
1093 if (adreno_is_a740_family(adreno_gpu) || adreno_is_a722(adreno_gpu) ||
1094 adreno_is_a8xx(adreno_gpu))
1095 seqmem_off = 4;
1096
1097 /* Make sure there are no outstanding RPMh votes */
1098 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS + seqmem_off,
1099 val, (val & 1), 100, 10000);
1100 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS + seqmem_off,
1101 val, (val & 1), 100, 10000);
1102 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS + seqmem_off,
1103 val, (val & 1), 100, 10000);
1104 gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS + seqmem_off,
1105 val, (val & 1), 100, 1000);
1106
1107 if (!adreno_is_a740_family(adreno_gpu) && !adreno_is_a722(adreno_gpu) &&
1108 !adreno_is_a8xx(adreno_gpu))
1109 return;
1110
1111 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS4_DRV0_STATUS + seqmem_off,
1112 val, (val & 1), 100, 10000);
1113 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS5_DRV0_STATUS + seqmem_off,
1114 val, (val & 1), 100, 10000);
1115 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS6_DRV0_STATUS + seqmem_off,
1116 val, (val & 1), 100, 10000);
1117 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS7_DRV0_STATUS + seqmem_off,
1118 val, (val & 1), 100, 1000);
1119 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS8_DRV0_STATUS + seqmem_off,
1120 val, (val & 1), 100, 10000);
1121 gmu_poll_timeout_rscc(gmu, REG_A7XX_RSCC_TCS9_DRV0_STATUS + seqmem_off,
1122 val, (val & 1), 100, 1000);
1123 }
1124
1125 /* Force the GMU off in case it isn't responsive */
a6xx_gmu_force_off(struct a6xx_gmu * gmu)1126 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
1127 {
1128 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1129 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1130 struct msm_gpu *gpu = &adreno_gpu->base;
1131
1132 /*
1133 * Turn off keep alive that might have been enabled by the hang
1134 * interrupt
1135 */
1136 if (adreno_is_a8xx(adreno_gpu))
1137 gmu_write(&a6xx_gpu->gmu, REG_A8XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
1138 else
1139 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
1140
1141 /* Flush all the queues */
1142 a6xx_hfi_stop(gmu);
1143
1144 /* Stop the interrupts */
1145 a6xx_gmu_irq_disable(gmu);
1146
1147 /* Force off SPTP in case the GMU is managing it */
1148 a6xx_sptprac_disable(gmu);
1149
1150 a6xx_gemnoc_workaround(gmu);
1151
1152 /* Make sure there are no outstanding RPMh votes */
1153 a6xx_gmu_rpmh_off(gmu);
1154
1155 /* Clear the WRITEDROPPED fields and put fence into allow mode */
1156 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7);
1157 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
1158
1159 /* Make sure the above writes go through */
1160 wmb();
1161
1162 /* Halt the gmu cm3 core */
1163 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
1164
1165 adreno_gpu->funcs->bus_halt(adreno_gpu, true);
1166
1167 /* Reset GPU core blocks */
1168 a6xx_gpu_sw_reset(gpu, true);
1169
1170 a6xx_rpmh_stop(gmu);
1171 }
1172
a6xx_gmu_set_initial_freq(struct msm_gpu * gpu,struct a6xx_gmu * gmu)1173 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
1174 {
1175 struct dev_pm_opp *gpu_opp;
1176 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
1177
1178 gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
1179 if (IS_ERR(gpu_opp))
1180 return;
1181
1182 gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */
1183 a6xx_gmu_set_freq(gpu, gpu_opp, false);
1184 dev_pm_opp_put(gpu_opp);
1185 }
1186
a6xx_gmu_set_initial_bw(struct msm_gpu * gpu,struct a6xx_gmu * gmu)1187 static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
1188 {
1189 struct dev_pm_opp *gpu_opp;
1190 unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
1191
1192 gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
1193 if (IS_ERR(gpu_opp))
1194 return;
1195
1196 dev_pm_opp_set_opp(&gpu->pdev->dev, gpu_opp);
1197 dev_pm_opp_put(gpu_opp);
1198 }
1199
a6xx_gmu_secure_init(struct a6xx_gpu * a6xx_gpu)1200 static int a6xx_gmu_secure_init(struct a6xx_gpu *a6xx_gpu)
1201 {
1202 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1203 struct msm_gpu *gpu = &adreno_gpu->base;
1204 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1205 u32 fuse_val;
1206 int ret;
1207
1208 if (test_bit(GMU_STATUS_SECURE_INIT, &gmu->status))
1209 return 0;
1210
1211 if (adreno_is_a750(adreno_gpu) || adreno_is_a8xx(adreno_gpu)) {
1212 /*
1213 * Assume that if qcom scm isn't available, that whatever
1214 * replacement allows writing the fuse register ourselves.
1215 * Users of alternative firmware need to make sure this
1216 * register is writeable or indicate that it's not somehow.
1217 * Print a warning because if you mess this up you're about to
1218 * crash horribly.
1219 */
1220 if (!qcom_scm_is_available()) {
1221 dev_warn_once(gpu->dev->dev,
1222 "SCM is not available, poking fuse register\n");
1223 a6xx_cx_misc_write(a6xx_gpu, REG_A7XX_CX_MISC_SW_FUSE_VALUE,
1224 A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING |
1225 A7XX_CX_MISC_SW_FUSE_VALUE_FASTBLEND |
1226 A7XX_CX_MISC_SW_FUSE_VALUE_LPAC);
1227 adreno_gpu->has_ray_tracing = true;
1228 goto done;
1229 }
1230
1231 ret = qcom_scm_gpu_init_regs(QCOM_SCM_GPU_ALWAYS_EN_REQ |
1232 QCOM_SCM_GPU_TSENSE_EN_REQ);
1233 if (ret) {
1234 dev_warn_once(gpu->dev->dev,
1235 "SCM call failed\n");
1236 return ret;
1237 }
1238
1239 /*
1240 * On A7XX_GEN3 and newer, raytracing may be disabled by the
1241 * firmware, find out whether that's the case. The scm call
1242 * above sets the fuse register.
1243 */
1244 fuse_val = a6xx_cx_misc_read(a6xx_gpu,
1245 REG_A7XX_CX_MISC_SW_FUSE_VALUE);
1246 adreno_gpu->has_ray_tracing =
1247 !!(fuse_val & A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING);
1248 } else if (adreno_is_a740(adreno_gpu)) {
1249 /* Raytracing is always enabled on a740 */
1250 adreno_gpu->has_ray_tracing = true;
1251 }
1252
1253 done:
1254 set_bit(GMU_STATUS_SECURE_INIT, &gmu->status);
1255 return 0;
1256 }
1257
a6xx_gmu_gxpd_get(struct a6xx_gmu * gmu)1258 static int a6xx_gmu_gxpd_get(struct a6xx_gmu *gmu)
1259 {
1260 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1261 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1262
1263 if (IS_ERR_OR_NULL(gmu->gxpd))
1264 return 0;
1265
1266 /*
1267 * On A8xx HW, GX GDSC is moved to a new clk controller block under GX
1268 * power domain. The clock driver for this new block keeps the GX rail
1269 * voted when gxpd is voted. So, use the gxpd only during gpu recovery.
1270 */
1271 if (adreno_gpu->info->family >= ADRENO_8XX_GEN1)
1272 return 0;
1273
1274 /*
1275 * On A6x/A7x, "enable" the GX power domain which won't actually do
1276 * anything but it will make sure that the refcounting is correct in
1277 * case we need to bring down the GX after a GMU failure
1278 */
1279 return pm_runtime_get_sync(gmu->gxpd);
1280 }
1281
a6xx_gmu_gxpd_put(struct a6xx_gmu * gmu)1282 static int a6xx_gmu_gxpd_put(struct a6xx_gmu *gmu)
1283 {
1284 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1285 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1286
1287 if (IS_ERR_OR_NULL(gmu->gxpd))
1288 return 0;
1289
1290 if (adreno_gpu->info->family < ADRENO_8XX_GEN1)
1291 return pm_runtime_put_sync(gmu->gxpd);
1292
1293 /*
1294 * On A8x, GX GDSC collapse should be triggered only when it is stuck ON
1295 */
1296 if (adreno_gpu->funcs->gx_is_on(adreno_gpu)) {
1297 pm_runtime_get_sync(gmu->gxpd);
1298 /*
1299 * Hint to gfxclkctl driver to do a hw collapse during the next
1300 * RPM PUT. This is a special behavior in the gfxclkctl driver
1301 */
1302 dev_pm_genpd_synced_poweroff(gmu->gxpd);
1303 pm_runtime_put_sync(gmu->gxpd);
1304 }
1305
1306 return 0;
1307 }
1308
a6xx_gmu_resume(struct a6xx_gpu * a6xx_gpu)1309 int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
1310 {
1311 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1312 struct msm_gpu *gpu = &adreno_gpu->base;
1313 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1314 int status, ret;
1315
1316 if (WARN(!gmu->initialized, "The GMU is not set up yet\n"))
1317 return -EINVAL;
1318
1319 gmu->hung = false;
1320
1321 /* Turn on the resources */
1322 pm_runtime_get_sync(gmu->dev);
1323
1324 a6xx_gmu_gxpd_get(gmu);
1325
1326 /* Use a known rate to bring up the GMU */
1327 clk_set_rate(gmu->core_clk, 200000000);
1328 clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ?
1329 200000000 : 150000000);
1330 ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
1331 if (ret)
1332 goto rpm_put;
1333
1334 ret = a6xx_gmu_secure_init(a6xx_gpu);
1335 if (ret)
1336 goto disable_clk;
1337
1338 /* Read the slice info on A8x GPUs */
1339 a8xx_gpu_get_slice_info(gpu);
1340
1341 /* Set the bus quota to a reasonable value for boot */
1342 a6xx_gmu_set_initial_bw(gpu, gmu);
1343
1344 /* Enable the GMU interrupt */
1345 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
1346 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK);
1347 enable_irq(gmu->gmu_irq);
1348
1349 /* Check to see if we are doing a cold or warm boot */
1350 if (adreno_is_a7xx(adreno_gpu) || adreno_is_a8xx(adreno_gpu)) {
1351 status = a6xx_cx_misc_read(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL) == 1 ?
1352 GMU_WARM_BOOT : GMU_COLD_BOOT;
1353 } else if (gmu->legacy) {
1354 status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
1355 GMU_WARM_BOOT : GMU_COLD_BOOT;
1356 } else {
1357 /*
1358 * Warm boot path does not work on newer A6xx GPUs
1359 * Presumably this is because icache/dcache regions must be restored
1360 */
1361 status = GMU_COLD_BOOT;
1362 }
1363
1364 ret = a6xx_gmu_fw_start(gmu, status);
1365 if (ret)
1366 goto disable_irq;
1367
1368 ret = a6xx_hfi_start(gmu, status);
1369 if (ret)
1370 goto disable_irq;
1371
1372 /*
1373 * Turn on the GMU firmware fault interrupt after we know the boot
1374 * sequence is successful
1375 */
1376 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
1377 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK);
1378 enable_irq(gmu->hfi_irq);
1379
1380 /* Set the GPU to the current freq */
1381 a6xx_gmu_set_initial_freq(gpu, gmu);
1382
1383 return 0;
1384
1385 disable_irq:
1386 disable_irq(gmu->gmu_irq);
1387 a6xx_rpmh_stop(gmu);
1388 disable_clk:
1389 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
1390 rpm_put:
1391 a6xx_gmu_gxpd_put(gmu);
1392
1393 pm_runtime_put(gmu->dev);
1394
1395 return ret;
1396 }
1397
a6xx_gmu_isidle(struct a6xx_gmu * gmu)1398 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
1399 {
1400 u32 reg;
1401
1402 if (!gmu->initialized)
1403 return true;
1404
1405 reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
1406
1407 if (reg & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB)
1408 return false;
1409
1410 return true;
1411 }
1412
1413 /* Gracefully try to shut down the GMU and by extension the GPU */
a6xx_gmu_shutdown(struct a6xx_gmu * gmu)1414 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
1415 {
1416 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1417 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1418 u32 val;
1419 int ret;
1420
1421 /*
1422 * GMU firmware's internal power state gets messed up if we send "prepare_slumber" hfi when
1423 * oob_gpu handshake wasn't done after the last wake up. So do a dummy handshake here when
1424 * required
1425 */
1426 if (adreno_gpu->base.needs_hw_init) {
1427 if (a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET))
1428 goto force_off;
1429
1430 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1431 }
1432
1433 if (test_and_clear_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status))
1434 a6xx_gmu_clear_oob(gmu, GMU_OOB_PERFCOUNTER_SET);
1435
1436 ret = a6xx_gmu_wait_for_idle(gmu);
1437
1438 /* If the GMU isn't responding assume it is hung */
1439 if (ret)
1440 goto force_off;
1441
1442 adreno_gpu->funcs->bus_halt(adreno_gpu, a6xx_gpu->hung);
1443
1444 /* tell the GMU we want to slumber */
1445 ret = a6xx_gmu_notify_slumber(gmu);
1446 if (ret)
1447 goto force_off;
1448
1449 ret = gmu_poll_timeout(gmu,
1450 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
1451 !(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB),
1452 100, 10000);
1453
1454 /*
1455 * Let the user know we failed to slumber but don't worry too
1456 * much because we are powering down anyway
1457 */
1458
1459 if (ret)
1460 DRM_DEV_ERROR(gmu->dev,
1461 "Unable to slumber GMU: status = 0%x/0%x\n",
1462 gmu_read(gmu,
1463 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
1464 gmu_read(gmu,
1465 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2));
1466
1467 /* Turn off HFI */
1468 a6xx_hfi_stop(gmu);
1469
1470 /* Stop the interrupts and mask the hardware */
1471 a6xx_gmu_irq_disable(gmu);
1472
1473 /* Halt the gmu cm3 core */
1474 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
1475
1476 /* Tell RPMh to power off the GPU */
1477 a6xx_rpmh_stop(gmu);
1478
1479 return;
1480
1481 force_off:
1482 a6xx_gmu_force_off(gmu);
1483 }
1484
1485
a6xx_gmu_stop(struct a6xx_gpu * a6xx_gpu)1486 int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
1487 {
1488 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1489 struct msm_gpu *gpu = &a6xx_gpu->base.base;
1490
1491 if (!pm_runtime_active(gmu->dev))
1492 return 0;
1493
1494 /*
1495 * Force the GMU off if we detected a hang, otherwise try to shut it
1496 * down gracefully
1497 */
1498 if (gmu->hung)
1499 a6xx_gmu_force_off(gmu);
1500 else
1501 a6xx_gmu_shutdown(gmu);
1502
1503 /* Remove the bus vote */
1504 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
1505
1506 /*
1507 * Make sure the GX domain is off before turning off the GMU (CX)
1508 * domain. Usually the GMU does this but only if the shutdown sequence
1509 * was successful
1510 */
1511 a6xx_gmu_gxpd_put(gmu);
1512
1513 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
1514
1515 pm_runtime_put_sync(gmu->dev);
1516
1517 return 0;
1518 }
1519
a6xx_gmu_memory_free(struct a6xx_gmu * gmu)1520 static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu)
1521 {
1522 struct msm_mmu *mmu = to_msm_vm(gmu->vm)->mmu;
1523
1524 msm_gem_kernel_put(gmu->hfi.obj, gmu->vm);
1525 msm_gem_kernel_put(gmu->debug.obj, gmu->vm);
1526 msm_gem_kernel_put(gmu->icache.obj, gmu->vm);
1527 msm_gem_kernel_put(gmu->dcache.obj, gmu->vm);
1528 msm_gem_kernel_put(gmu->dummy.obj, gmu->vm);
1529 msm_gem_kernel_put(gmu->log.obj, gmu->vm);
1530
1531 mmu->funcs->detach(mmu);
1532 drm_gpuvm_put(gmu->vm);
1533 }
1534
a6xx_gmu_memory_alloc(struct a6xx_gmu * gmu,struct a6xx_gmu_bo * bo,size_t size,u64 iova,const char * name)1535 static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo,
1536 size_t size, u64 iova, const char *name)
1537 {
1538 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1539 struct drm_device *dev = a6xx_gpu->base.base.dev;
1540 uint32_t flags = MSM_BO_WC;
1541 u64 range_start, range_end;
1542 int ret;
1543
1544 size = PAGE_ALIGN(size);
1545 if (!iova) {
1546 /* no fixed address - use GMU's uncached range */
1547 range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */
1548 range_end = 0x80000000;
1549 } else {
1550 /* range for fixed address */
1551 range_start = iova;
1552 range_end = iova + size;
1553 /* use IOMMU_PRIV for icache/dcache */
1554 flags |= MSM_BO_MAP_PRIV;
1555 }
1556
1557 bo->obj = msm_gem_new(dev, size, flags, NULL);
1558 if (IS_ERR(bo->obj))
1559 return PTR_ERR(bo->obj);
1560
1561 ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->vm, &bo->iova,
1562 range_start, range_end);
1563 if (ret) {
1564 drm_gem_object_put(bo->obj);
1565 return ret;
1566 }
1567
1568 bo->virt = msm_gem_get_vaddr(bo->obj);
1569 bo->size = size;
1570
1571 msm_gem_object_set_name(bo->obj, "%s", name);
1572
1573 return 0;
1574 }
1575
a6xx_gmu_memory_probe(struct drm_device * drm,struct a6xx_gmu * gmu)1576 static int a6xx_gmu_memory_probe(struct drm_device *drm, struct a6xx_gmu *gmu)
1577 {
1578 struct msm_mmu *mmu;
1579
1580 mmu = msm_iommu_new(gmu->dev, 0);
1581 if (IS_ERR(mmu))
1582 return PTR_ERR(mmu);
1583
1584 gmu->vm = msm_gem_vm_create(drm, mmu, "gmu", 0x0, 0x80000000, true);
1585 if (IS_ERR(gmu->vm))
1586 return PTR_ERR(gmu->vm);
1587
1588 return 0;
1589 }
1590
1591 /**
1592 * struct bcm_db - Auxiliary data pertaining to each Bus Clock Manager (BCM)
1593 * @unit: divisor used to convert bytes/sec bw value to an RPMh msg
1594 * @width: multiplier used to convert bytes/sec bw value to an RPMh msg
1595 * @vcd: virtual clock domain that this bcm belongs to
1596 * @reserved: reserved field
1597 */
1598 struct bcm_db {
1599 __le32 unit;
1600 __le16 width;
1601 u8 vcd;
1602 u8 reserved;
1603 };
1604
a6xx_gmu_rpmh_bw_votes_init(struct adreno_gpu * adreno_gpu,const struct a6xx_info * info,struct a6xx_gmu * gmu)1605 static int a6xx_gmu_rpmh_bw_votes_init(struct adreno_gpu *adreno_gpu,
1606 const struct a6xx_info *info,
1607 struct a6xx_gmu *gmu)
1608 {
1609 const struct bcm_db *bcm_data[GMU_MAX_BCMS] = { 0 };
1610 unsigned int bcm_index, bw_index, bcm_count = 0;
1611
1612 /* Retrieve BCM data from cmd-db */
1613 for (bcm_index = 0; bcm_index < GMU_MAX_BCMS; bcm_index++) {
1614 const struct a6xx_bcm *bcm = &info->bcms[bcm_index];
1615 size_t count;
1616
1617 /* Stop at NULL terminated bcm entry */
1618 if (!bcm->name)
1619 break;
1620
1621 bcm_data[bcm_index] = cmd_db_read_aux_data(bcm->name, &count);
1622 if (IS_ERR(bcm_data[bcm_index]))
1623 return PTR_ERR(bcm_data[bcm_index]);
1624
1625 if (!count) {
1626 dev_err(gmu->dev, "invalid BCM '%s' aux data size\n",
1627 bcm->name);
1628 return -EINVAL;
1629 }
1630
1631 bcm_count++;
1632 }
1633
1634 /* Generate BCM votes values for each bandwidth & BCM */
1635 for (bw_index = 0; bw_index < gmu->nr_gpu_bws; bw_index++) {
1636 u32 *data = gmu->gpu_ib_votes[bw_index];
1637 u32 bw = gmu->gpu_bw_table[bw_index];
1638
1639 /* Calculations loosely copied from bcm_aggregate() & tcs_cmd_gen() */
1640 for (bcm_index = 0; bcm_index < bcm_count; bcm_index++) {
1641 const struct a6xx_bcm *bcm = &info->bcms[bcm_index];
1642 bool commit = false;
1643 u64 peak;
1644 u32 vote;
1645
1646 if (bcm_index == bcm_count - 1 ||
1647 (bcm_data[bcm_index + 1] &&
1648 bcm_data[bcm_index]->vcd != bcm_data[bcm_index + 1]->vcd))
1649 commit = true;
1650
1651 if (!bw) {
1652 data[bcm_index] = BCM_TCS_CMD(commit, false, 0, 0);
1653 continue;
1654 }
1655
1656 if (bcm->fixed) {
1657 u32 perfmode = 0;
1658
1659 /* GMU on A6xx votes perfmode on all valid bandwidth */
1660 if (!adreno_is_a7xx(adreno_gpu) ||
1661 (bcm->perfmode_bw && bw >= bcm->perfmode_bw))
1662 perfmode = bcm->perfmode;
1663
1664 data[bcm_index] = BCM_TCS_CMD(commit, true, 0, perfmode);
1665 continue;
1666 }
1667
1668 /* Multiply the bandwidth by the width of the connection */
1669 peak = (u64)bw * le16_to_cpu(bcm_data[bcm_index]->width);
1670 do_div(peak, bcm->buswidth);
1671
1672 /* Input bandwidth value is in KBps, scale the value to BCM unit */
1673 peak *= 1000;
1674 do_div(peak, le32_to_cpu(bcm_data[bcm_index]->unit));
1675
1676 vote = clamp(peak, 1, BCM_TCS_CMD_VOTE_MASK);
1677
1678 /* GMUs on A7xx votes on both x & y */
1679 if (adreno_is_a7xx(adreno_gpu) || adreno_is_a8xx(adreno_gpu))
1680 data[bcm_index] = BCM_TCS_CMD(commit, true, vote, vote);
1681 else
1682 data[bcm_index] = BCM_TCS_CMD(commit, true, 0, vote);
1683 }
1684 }
1685
1686 return 0;
1687 }
1688
1689 /* Return the 'arc-level' for the given frequency */
a6xx_gmu_get_arc_level(struct device * dev,unsigned long freq)1690 static unsigned int a6xx_gmu_get_arc_level(struct device *dev,
1691 unsigned long freq)
1692 {
1693 struct dev_pm_opp *opp;
1694 unsigned int val;
1695
1696 if (!freq)
1697 return 0;
1698
1699 opp = dev_pm_opp_find_freq_exact(dev, freq, true);
1700 if (IS_ERR(opp))
1701 return 0;
1702
1703 val = dev_pm_opp_get_level(opp);
1704
1705 dev_pm_opp_put(opp);
1706
1707 return val;
1708 }
1709
a6xx_gmu_rpmh_arc_votes_init(struct device * dev,u32 * votes,unsigned long * freqs,int freqs_count,const char * pri_id,const char * sec_id)1710 static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
1711 unsigned long *freqs, int freqs_count,
1712 const char *pri_id, const char *sec_id)
1713 {
1714 int i, j;
1715 const u16 *pri, *sec;
1716 size_t pri_count, sec_count;
1717
1718 pri = cmd_db_read_aux_data(pri_id, &pri_count);
1719 if (IS_ERR(pri))
1720 return PTR_ERR(pri);
1721 /*
1722 * The data comes back as an array of unsigned shorts so adjust the
1723 * count accordingly
1724 */
1725 pri_count >>= 1;
1726 if (!pri_count)
1727 return -EINVAL;
1728
1729 sec = cmd_db_read_aux_data(sec_id, &sec_count);
1730 if (IS_ERR(sec))
1731 return PTR_ERR(sec);
1732
1733 sec_count >>= 1;
1734 if (!sec_count)
1735 return -EINVAL;
1736
1737 /* Construct a vote for each frequency */
1738 for (i = 0; i < freqs_count; i++) {
1739 u8 pindex = 0, sindex = 0;
1740 unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
1741
1742 /* Get the primary index that matches the arc level */
1743 for (j = 0; j < pri_count; j++) {
1744 if (pri[j] >= level) {
1745 pindex = j;
1746 break;
1747 }
1748 }
1749
1750 if (j == pri_count) {
1751 DRM_DEV_ERROR(dev,
1752 "Level %u not found in the RPMh list\n",
1753 level);
1754 DRM_DEV_ERROR(dev, "Available levels:\n");
1755 for (j = 0; j < pri_count; j++)
1756 DRM_DEV_ERROR(dev, " %u\n", pri[j]);
1757
1758 return -EINVAL;
1759 }
1760
1761 /*
1762 * Look for a level in in the secondary list that matches. If
1763 * nothing fits, use the maximum non zero vote
1764 */
1765
1766 for (j = 0; j < sec_count; j++) {
1767 if (sec[j] >= level) {
1768 sindex = j;
1769 break;
1770 } else if (sec[j]) {
1771 sindex = j;
1772 }
1773 }
1774
1775 /* Construct the vote */
1776 votes[i] = ((pri[pindex] & 0xffff) << 16) |
1777 (sindex << 8) | pindex;
1778 }
1779
1780 return 0;
1781 }
1782
a6xx_gmu_rpmh_dep_votes_init(struct device * dev,u32 * votes,unsigned long * freqs,int freqs_count)1783 static int a6xx_gmu_rpmh_dep_votes_init(struct device *dev, u32 *votes,
1784 unsigned long *freqs, int freqs_count)
1785 {
1786 const u16 *mx;
1787 size_t count;
1788
1789 mx = cmd_db_read_aux_data("mx.lvl", &count);
1790 if (IS_ERR(mx))
1791 return PTR_ERR(mx);
1792 /*
1793 * The data comes back as an array of unsigned shorts so adjust the
1794 * count accordingly
1795 */
1796 count >>= 1;
1797 if (!count)
1798 return -EINVAL;
1799
1800 /* Fix the vote for zero frequency */
1801 votes[0] = 0xffffffff;
1802
1803 /* Construct a vote for rest of the corners */
1804 for (int i = 1; i < freqs_count; i++) {
1805 unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
1806 u8 j, index = 0;
1807
1808 /* Get the primary index that matches the arc level */
1809 for (j = 0; j < count; j++) {
1810 if (mx[j] >= level) {
1811 index = j;
1812 break;
1813 }
1814 }
1815
1816 if (j == count) {
1817 DRM_DEV_ERROR(dev,
1818 "Mx Level %u not found in the RPMh list\n",
1819 level);
1820 DRM_DEV_ERROR(dev, "Available levels:\n");
1821 for (j = 0; j < count; j++)
1822 DRM_DEV_ERROR(dev, " %u\n", mx[j]);
1823
1824 return -EINVAL;
1825 }
1826
1827 /* Construct the vote */
1828 votes[i] = (0x3fff << 14) | (index << 8) | (0xff);
1829 }
1830
1831 return 0;
1832 }
1833
1834 /*
1835 * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
1836 * to construct the list of votes on the CPU and send it over. Query the RPMh
1837 * voltage levels and build the votes
1838 * The GMU can also vote for DDR interconnects, use the OPP bandwidth entries
1839 * and BCM parameters to build the votes.
1840 */
1841
a6xx_gmu_rpmh_votes_init(struct a6xx_gmu * gmu)1842 static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu)
1843 {
1844 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1845 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1846 const struct a6xx_info *info = adreno_gpu->info->a6xx;
1847 struct msm_gpu *gpu = &adreno_gpu->base;
1848 const char *sec_id;
1849 const u16 *gmxc;
1850 int ret;
1851
1852 gmxc = cmd_db_read_aux_data("gmxc.lvl", NULL);
1853 if (gmxc == ERR_PTR(-EPROBE_DEFER))
1854 return -EPROBE_DEFER;
1855
1856 /* If GMxC is present, prefer that as secondary rail for GX votes */
1857 sec_id = IS_ERR_OR_NULL(gmxc) ? "mx.lvl" : "gmxc.lvl";
1858
1859 /* Build the GX votes */
1860 ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
1861 gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl", sec_id);
1862
1863 /* Build the CX votes */
1864 ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
1865 gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl", "mx.lvl");
1866
1867 ret |= a6xx_gmu_rpmh_dep_votes_init(gmu->dev, gmu->dep_arc_votes,
1868 gmu->gpu_freqs, gmu->nr_gpu_freqs);
1869
1870 /* Build the interconnect votes */
1871 if (info->bcms && gmu->nr_gpu_bws > 1)
1872 ret |= a6xx_gmu_rpmh_bw_votes_init(adreno_gpu, info, gmu);
1873
1874 return ret;
1875 }
1876
a6xx_gmu_build_freq_table(struct device * dev,unsigned long * freqs,u32 size)1877 static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs,
1878 u32 size)
1879 {
1880 int count = dev_pm_opp_get_opp_count(dev);
1881 struct dev_pm_opp *opp;
1882 int i, index = 0;
1883 unsigned long freq = 1;
1884
1885 /*
1886 * The OPP table doesn't contain the "off" frequency level so we need to
1887 * add 1 to the table size to account for it
1888 */
1889
1890 if (WARN(count + 1 > size,
1891 "The GMU frequency table is being truncated\n"))
1892 count = size - 1;
1893
1894 /* Set the "off" frequency */
1895 freqs[index++] = 0;
1896
1897 for (i = 0; i < count; i++) {
1898 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
1899 if (IS_ERR(opp))
1900 break;
1901
1902 dev_pm_opp_put(opp);
1903 freqs[index++] = freq++;
1904 }
1905
1906 return index;
1907 }
1908
a6xx_gmu_build_bw_table(struct device * dev,unsigned long * bandwidths,u32 size)1909 static int a6xx_gmu_build_bw_table(struct device *dev, unsigned long *bandwidths,
1910 u32 size)
1911 {
1912 int count = dev_pm_opp_get_opp_count(dev);
1913 struct dev_pm_opp *opp;
1914 int i, index = 0;
1915 unsigned int bandwidth = 1;
1916
1917 /*
1918 * The OPP table doesn't contain the "off" bandwidth level so we need to
1919 * add 1 to the table size to account for it
1920 */
1921
1922 if (WARN(count + 1 > size,
1923 "The GMU bandwidth table is being truncated\n"))
1924 count = size - 1;
1925
1926 /* Set the "off" bandwidth */
1927 bandwidths[index++] = 0;
1928
1929 for (i = 0; i < count; i++) {
1930 opp = dev_pm_opp_find_bw_ceil(dev, &bandwidth, 0);
1931 if (IS_ERR(opp))
1932 break;
1933
1934 dev_pm_opp_put(opp);
1935 bandwidths[index++] = bandwidth++;
1936 }
1937
1938 return index;
1939 }
1940
a6xx_gmu_pwrlevels_probe(struct a6xx_gmu * gmu)1941 static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu)
1942 {
1943 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1944 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1945 const struct a6xx_info *info = adreno_gpu->info->a6xx;
1946 struct msm_gpu *gpu = &adreno_gpu->base;
1947
1948 int ret = 0;
1949
1950 /*
1951 * The GMU handles its own frequency switching so build a list of
1952 * available frequencies to send during initialization
1953 */
1954 ret = devm_pm_opp_of_add_table(gmu->dev);
1955 if (ret) {
1956 DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n");
1957 return ret;
1958 }
1959
1960 gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev,
1961 gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs));
1962
1963 /*
1964 * The GMU also handles GPU frequency switching so build a list
1965 * from the GPU OPP table
1966 */
1967 gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
1968 gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
1969
1970 gmu->current_perf_index = gmu->nr_gpu_freqs - 1;
1971
1972 /*
1973 * The GMU also handles GPU Interconnect Votes so build a list
1974 * of DDR bandwidths from the GPU OPP table
1975 */
1976 if (info->bcms)
1977 gmu->nr_gpu_bws = a6xx_gmu_build_bw_table(&gpu->pdev->dev,
1978 gmu->gpu_bw_table, ARRAY_SIZE(gmu->gpu_bw_table));
1979
1980 /* Build the list of RPMh votes that we'll send to the GMU */
1981 return a6xx_gmu_rpmh_votes_init(gmu);
1982 }
1983
a6xx_gmu_acd_probe(struct a6xx_gmu * gmu)1984 static int a6xx_gmu_acd_probe(struct a6xx_gmu *gmu)
1985 {
1986 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1987 struct a6xx_hfi_acd_table *cmd = &gmu->acd_table;
1988 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1989 struct msm_gpu *gpu = &adreno_gpu->base;
1990 int ret, i, cmd_idx = 0;
1991 extern bool disable_acd;
1992
1993 /* Skip ACD probe if requested via module param */
1994 if (disable_acd) {
1995 DRM_DEV_ERROR(gmu->dev, "Skipping GPU ACD probe\n");
1996 return 0;
1997 }
1998
1999 cmd->version = 1;
2000 cmd->stride = 1;
2001 cmd->enable_by_level = 0;
2002
2003 /* Skip freq = 0 and parse acd-level for rest of the OPPs */
2004 for (i = 1; i < gmu->nr_gpu_freqs; i++) {
2005 struct dev_pm_opp *opp;
2006 struct device_node *np;
2007 unsigned long freq;
2008 u32 val;
2009
2010 freq = gmu->gpu_freqs[i];
2011 /* This is unlikely to fail because we are passing back a known freq */
2012 opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, freq, true);
2013 np = dev_pm_opp_get_of_node(opp);
2014
2015 ret = of_property_read_u32(np, "qcom,opp-acd-level", &val);
2016 of_node_put(np);
2017 dev_pm_opp_put(opp);
2018 if (ret == -EINVAL)
2019 continue;
2020 else if (ret) {
2021 DRM_DEV_ERROR(gmu->dev, "Unable to read acd level for freq %lu\n", freq);
2022 return ret;
2023 }
2024
2025 cmd->enable_by_level |= BIT(i);
2026 cmd->data[cmd_idx++] = val;
2027 }
2028
2029 cmd->num_levels = cmd_idx;
2030
2031 /* It is a problem if qmp node is unavailable when ACD is required */
2032 if (cmd->enable_by_level && IS_ERR_OR_NULL(gmu->qmp)) {
2033 DRM_DEV_ERROR(gmu->dev, "Unable to send ACD state to AOSS\n");
2034 return -EINVAL;
2035 }
2036
2037 /* Otherwise, nothing to do if qmp is unavailable */
2038 if (IS_ERR_OR_NULL(gmu->qmp))
2039 return 0;
2040
2041 /*
2042 * Notify AOSS about the ACD state. AOSS is supposed to assume that ACD is disabled on
2043 * system reset. So it is harmless if we couldn't notify 'OFF' state
2044 */
2045 ret = qmp_send(gmu->qmp, "{class: gpu, res: acd, val: %d}", !!cmd->enable_by_level);
2046 if (ret && cmd->enable_by_level) {
2047 DRM_DEV_ERROR(gmu->dev, "Failed to send ACD state to AOSS\n");
2048 return ret;
2049 }
2050
2051 return 0;
2052 }
2053
a6xx_gmu_clocks_probe(struct a6xx_gmu * gmu)2054 static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
2055 {
2056 int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks);
2057
2058 if (ret < 1)
2059 return ret;
2060
2061 gmu->nr_clocks = ret;
2062
2063 gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks,
2064 gmu->nr_clocks, "gmu");
2065
2066 gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks,
2067 gmu->nr_clocks, "hub");
2068
2069 return 0;
2070 }
2071
a6xx_gmu_get_irq(struct a6xx_gmu * gmu,struct platform_device * pdev,const char * name,irq_handler_t handler)2072 static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
2073 const char *name, irq_handler_t handler)
2074 {
2075 int irq, ret;
2076
2077 irq = platform_get_irq_byname(pdev, name);
2078
2079 ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN, name, gmu);
2080 if (ret) {
2081 DRM_DEV_ERROR(&pdev->dev, "Unable to get interrupt %s %d\n",
2082 name, ret);
2083 return ret;
2084 }
2085
2086 return irq;
2087 }
2088
a6xx_gmu_sysprof_setup(struct msm_gpu * gpu,bool force_on)2089 void a6xx_gmu_sysprof_setup(struct msm_gpu *gpu, bool force_on)
2090 {
2091 bool sysprof = msm_gpu_sysprof_no_ifpc(gpu) || force_on;
2092 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2093 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2094 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2095
2096 /* Nothing to do if GPU is suspended. We will handle this during GMU resume */
2097 if (!pm_runtime_get_if_active(&gpu->pdev->dev))
2098 return;
2099
2100 mutex_lock(&gmu->lock);
2101
2102 /*
2103 * 'Perfcounter select' register values are lost during IFPC collapse. To avoid that,
2104 * use the currently unused perfcounter oob vote to block IFPC when sysprof is active
2105 */
2106 if (sysprof && !test_and_set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status))
2107 a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET);
2108 else if (!sysprof && test_and_clear_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status))
2109 a6xx_gmu_clear_oob(gmu, GMU_OOB_PERFCOUNTER_SET);
2110
2111 mutex_unlock(&gmu->lock);
2112
2113 pm_runtime_put(&gpu->pdev->dev);
2114 }
2115
a6xx_gmu_remove(struct a6xx_gpu * a6xx_gpu)2116 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
2117 {
2118 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2119 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2120
2121 mutex_lock(&gmu->lock);
2122 if (!gmu->initialized) {
2123 mutex_unlock(&gmu->lock);
2124 return;
2125 }
2126
2127 gmu->initialized = false;
2128
2129 mutex_unlock(&gmu->lock);
2130
2131 pm_runtime_force_suspend(gmu->dev);
2132
2133 /*
2134 * Since cxpd is a virt device, the devlink with gmu-dev will be removed
2135 * automatically when we do detach
2136 */
2137 dev_pm_domain_detach(gmu->cxpd, false);
2138
2139 if (!IS_ERR_OR_NULL(gmu->gxpd)) {
2140 pm_runtime_disable(gmu->gxpd);
2141 dev_pm_domain_detach(gmu->gxpd, false);
2142 }
2143
2144 if (!IS_ERR_OR_NULL(gmu->qmp))
2145 qmp_put(gmu->qmp);
2146
2147 iounmap(gmu->mmio);
2148 gmu->mmio = NULL;
2149 gmu->rscc = NULL;
2150
2151 if (!adreno_has_gmu_wrapper(adreno_gpu) &&
2152 !adreno_has_rgmu(adreno_gpu)) {
2153 a6xx_gmu_memory_free(gmu);
2154
2155 free_irq(gmu->gmu_irq, gmu);
2156 free_irq(gmu->hfi_irq, gmu);
2157 }
2158
2159 /* Drop reference taken in of_find_device_by_node */
2160 put_device(gmu->dev);
2161 }
2162
cxpd_notifier_cb(struct notifier_block * nb,unsigned long action,void * data)2163 static int cxpd_notifier_cb(struct notifier_block *nb,
2164 unsigned long action, void *data)
2165 {
2166 struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb);
2167
2168 if (action == GENPD_NOTIFY_OFF)
2169 complete_all(&gmu->pd_gate);
2170
2171 return 0;
2172 }
2173
a6xx_gmu_get_mmio(struct platform_device * pdev,resource_size_t * start)2174 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, resource_size_t *start)
2175 {
2176 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2177 void __iomem *ret;
2178
2179 if (!res) {
2180 DRM_DEV_ERROR(&pdev->dev, "Unable to find the gmu core registers\n");
2181 return ERR_PTR(-EINVAL);
2182 }
2183
2184 ret = ioremap(res->start, resource_size(res));
2185 if (!ret) {
2186 DRM_DEV_ERROR(&pdev->dev, "Unable to map the gmu core registers\n");
2187 return ERR_PTR(-EINVAL);
2188 }
2189
2190 if (start)
2191 *start = res->start;
2192
2193 return ret;
2194 }
2195
a6xx_gmu_wrapper_init(struct a6xx_gpu * a6xx_gpu,struct device_node * node)2196 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
2197 {
2198 struct platform_device *pdev = of_find_device_by_node(node);
2199 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2200 struct msm_gpu *gpu = &adreno_gpu->base;
2201 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2202 resource_size_t start;
2203 struct resource *res;
2204 int ret;
2205
2206 if (!pdev)
2207 return -ENODEV;
2208
2209 gmu->dev = &pdev->dev;
2210
2211 ret = of_dma_configure(gmu->dev, node, true);
2212 if (ret)
2213 return ret;
2214
2215 pm_runtime_enable(gmu->dev);
2216
2217 /* Mark legacy for manual SPTPRAC control */
2218 gmu->legacy = true;
2219
2220 /* RGMU requires clocks */
2221 ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks);
2222 if (ret < 0)
2223 goto err_clk;
2224
2225 gmu->nr_clocks = ret;
2226
2227 /* Map the GMU registers */
2228 gmu->mmio = a6xx_gmu_get_mmio(pdev, &start);
2229 if (IS_ERR(gmu->mmio)) {
2230 ret = PTR_ERR(gmu->mmio);
2231 goto err_mmio;
2232 }
2233
2234 res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory");
2235 if (!res) {
2236 ret = -EINVAL;
2237 goto err_mmio;
2238 }
2239
2240 /* Identify gmu base offset from gpu base address */
2241 gmu->mmio_offset = (u32)(start - res->start);
2242
2243 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
2244 if (IS_ERR(gmu->cxpd)) {
2245 ret = PTR_ERR(gmu->cxpd);
2246 goto err_mmio;
2247 }
2248
2249 if (!device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME)) {
2250 ret = -ENODEV;
2251 goto detach_cxpd;
2252 }
2253
2254 init_completion(&gmu->pd_gate);
2255 complete_all(&gmu->pd_gate);
2256 gmu->pd_nb.notifier_call = cxpd_notifier_cb;
2257
2258 /* Get a link to the GX power domain to reset the GPU */
2259 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
2260 if (IS_ERR(gmu->gxpd)) {
2261 ret = PTR_ERR(gmu->gxpd);
2262 goto err_mmio;
2263 }
2264
2265 gmu->initialized = true;
2266
2267 return 0;
2268
2269 detach_cxpd:
2270 dev_pm_domain_detach(gmu->cxpd, false);
2271
2272 err_mmio:
2273 iounmap(gmu->mmio);
2274
2275 err_clk:
2276 /* Drop reference taken in of_find_device_by_node */
2277 put_device(gmu->dev);
2278
2279 return ret;
2280 }
2281
a6xx_gmu_init(struct a6xx_gpu * a6xx_gpu,struct device_node * node)2282 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
2283 {
2284 struct platform_device *pdev = of_find_device_by_node(node);
2285 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2286 struct msm_gpu *gpu = &adreno_gpu->base;
2287 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2288 struct device_link *link;
2289 resource_size_t start;
2290 struct resource *res;
2291 int ret;
2292
2293 if (!pdev)
2294 return -ENODEV;
2295
2296 gmu->dev = &pdev->dev;
2297
2298 ret = of_dma_configure(gmu->dev, node, true);
2299 if (ret)
2300 return ret;
2301
2302 /* Set GMU idle level */
2303 gmu->idle_level = (adreno_gpu->info->quirks & ADRENO_QUIRK_IFPC) ?
2304 GMU_IDLE_STATE_IFPC : GMU_IDLE_STATE_ACTIVE;
2305
2306 pm_runtime_enable(gmu->dev);
2307
2308 /* Get the list of clocks */
2309 ret = a6xx_gmu_clocks_probe(gmu);
2310 if (ret)
2311 goto err_put_device;
2312
2313 ret = a6xx_gmu_memory_probe(adreno_gpu->base.dev, gmu);
2314 if (ret)
2315 goto err_put_device;
2316
2317
2318 /* A660 now requires handling "prealloc requests" in GMU firmware
2319 * For now just hardcode allocations based on the known firmware.
2320 * note: there is no indication that these correspond to "dummy" or
2321 * "debug" regions, but this "guess" allows reusing these BOs which
2322 * are otherwise unused by a660.
2323 */
2324 gmu->dummy.size = SZ_4K;
2325 if (adreno_is_a660_family(adreno_gpu) ||
2326 adreno_is_a7xx(adreno_gpu) ||
2327 adreno_is_a8xx(adreno_gpu)) {
2328 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7,
2329 0x60400000, "debug");
2330 if (ret)
2331 goto err_memory;
2332
2333 gmu->dummy.size = SZ_16K;
2334 }
2335
2336 /* Allocate memory for the GMU dummy page */
2337 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, gmu->dummy.size,
2338 0x60000000, "dummy");
2339 if (ret)
2340 goto err_memory;
2341
2342 /* Note that a650 family also includes a660 family: */
2343 if (adreno_is_a650_family(adreno_gpu) ||
2344 adreno_is_a7xx(adreno_gpu) ||
2345 adreno_is_a8xx(adreno_gpu)) {
2346 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
2347 SZ_16M - SZ_16K, 0x04000, "icache");
2348 if (ret)
2349 goto err_memory;
2350 /*
2351 * NOTE: when porting legacy ("pre-650-family") GPUs you may be tempted to add a condition
2352 * to allocate icache/dcache here, as per downstream code flow, but it may not actually be
2353 * necessary. If you omit this step and you don't get random pagefaults, you are likely
2354 * good to go without this!
2355 */
2356 } else if (adreno_is_a640_family(adreno_gpu)) {
2357 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
2358 SZ_256K - SZ_16K, 0x04000, "icache");
2359 if (ret)
2360 goto err_memory;
2361
2362 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache,
2363 SZ_256K - SZ_16K, 0x44000, "dcache");
2364 if (ret)
2365 goto err_memory;
2366 } else if (adreno_is_a630_family(adreno_gpu)) {
2367 /* HFI v1, has sptprac */
2368 gmu->legacy = true;
2369
2370 /* Allocate memory for the GMU debug region */
2371 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0, "debug");
2372 if (ret)
2373 goto err_memory;
2374 }
2375
2376 /* Allocate memory for the GMU log region */
2377 ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_16K, 0, "log");
2378 if (ret)
2379 goto err_memory;
2380
2381 /* Allocate memory for for the HFI queues */
2382 ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0, "hfi");
2383 if (ret)
2384 goto err_memory;
2385
2386 /* Map the GMU registers */
2387 gmu->mmio = a6xx_gmu_get_mmio(pdev, &start);
2388 if (IS_ERR(gmu->mmio)) {
2389 ret = PTR_ERR(gmu->mmio);
2390 goto err_memory;
2391 }
2392
2393 res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory");
2394 if (!res) {
2395 ret = -EINVAL;
2396 goto err_mmio;
2397 }
2398
2399 /* Identify gmu base offset from gpu base address */
2400 gmu->mmio_offset = (u32)(start - res->start);
2401
2402 if (adreno_is_a650_family(adreno_gpu) ||
2403 adreno_is_a7xx(adreno_gpu)) {
2404 gmu->rscc = devm_platform_ioremap_resource_byname(pdev, "rscc");
2405 if (IS_ERR(gmu->rscc)) {
2406 ret = -ENODEV;
2407 goto err_mmio;
2408 }
2409 } else if (adreno_is_a8xx(adreno_gpu)) {
2410 /*
2411 * On a8xx , RSCC lives at GPU base + 0x50000, which falls
2412 * inside the GPU's kgsl_3d0_reg_memory range rather than the
2413 * GMU's.
2414 */
2415 gmu->rscc = gpu->mmio + 0x50000;
2416 } else {
2417 gmu->rscc = gmu->mmio + 0x23000;
2418 }
2419
2420 /* Get the HFI and GMU interrupts */
2421 gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
2422 gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
2423
2424 if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) {
2425 ret = -ENODEV;
2426 goto err_mmio;
2427 }
2428
2429 gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
2430 if (IS_ERR(gmu->cxpd)) {
2431 ret = PTR_ERR(gmu->cxpd);
2432 goto err_mmio;
2433 }
2434
2435 link = device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME);
2436 if (!link) {
2437 ret = -ENODEV;
2438 goto detach_cxpd;
2439 }
2440
2441 /* Other errors are handled during GPU ACD probe */
2442 gmu->qmp = qmp_get(gmu->dev);
2443 if (PTR_ERR_OR_ZERO(gmu->qmp) == -EPROBE_DEFER) {
2444 ret = -EPROBE_DEFER;
2445 goto detach_gxpd;
2446 }
2447
2448 init_completion(&gmu->pd_gate);
2449 complete_all(&gmu->pd_gate);
2450 gmu->pd_nb.notifier_call = cxpd_notifier_cb;
2451
2452 /*
2453 * Get a link to the GX power domain to reset the GPU in case of GMU
2454 * crash
2455 */
2456 gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
2457
2458 /* Get the power levels for the GMU and GPU */
2459 a6xx_gmu_pwrlevels_probe(gmu);
2460
2461 ret = a6xx_gmu_acd_probe(gmu);
2462 if (ret)
2463 goto detach_gxpd;
2464
2465 /* Set up the HFI queues */
2466 a6xx_hfi_init(gmu);
2467
2468 /* Initialize RPMh */
2469 a6xx_gmu_rpmh_init(gmu);
2470
2471 gmu->initialized = true;
2472
2473 return 0;
2474
2475 detach_gxpd:
2476 if (!IS_ERR_OR_NULL(gmu->gxpd))
2477 dev_pm_domain_detach(gmu->gxpd, false);
2478
2479 if (!IS_ERR_OR_NULL(gmu->qmp))
2480 qmp_put(gmu->qmp);
2481
2482 device_link_del(link);
2483
2484 detach_cxpd:
2485 dev_pm_domain_detach(gmu->cxpd, false);
2486
2487 err_mmio:
2488 iounmap(gmu->mmio);
2489 free_irq(gmu->gmu_irq, gmu);
2490 free_irq(gmu->hfi_irq, gmu);
2491
2492 err_memory:
2493 a6xx_gmu_memory_free(gmu);
2494 err_put_device:
2495 /* Drop reference taken in of_find_device_by_node */
2496 put_device(gmu->dev);
2497
2498 return ret;
2499 }
2500