xref: /linux/drivers/gpu/drm/msm/adreno/a6xx_gmu.c (revision 170aafe35cb98e0f3fbacb446ea86389fbce22ea)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
3 
4 #include <linux/bitfield.h>
5 #include <linux/clk.h>
6 #include <linux/interconnect.h>
7 #include <linux/of_platform.h>
8 #include <linux/platform_device.h>
9 #include <linux/pm_domain.h>
10 #include <linux/pm_opp.h>
11 #include <soc/qcom/cmd-db.h>
12 #include <drm/drm_gem.h>
13 
14 #include "a6xx_gpu.h"
15 #include "a6xx_gmu.xml.h"
16 #include "msm_gem.h"
17 #include "msm_gpu_trace.h"
18 #include "msm_mmu.h"
19 
20 static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
21 {
22 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
23 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
24 	struct msm_gpu *gpu = &adreno_gpu->base;
25 
26 	/* FIXME: add a banner here */
27 	gmu->hung = true;
28 
29 	/* Turn off the hangcheck timer while we are resetting */
30 	del_timer(&gpu->hangcheck_timer);
31 
32 	/* Queue the GPU handler because we need to treat this as a recovery */
33 	kthread_queue_work(gpu->worker, &gpu->recover_work);
34 }
35 
36 static irqreturn_t a6xx_gmu_irq(int irq, void *data)
37 {
38 	struct a6xx_gmu *gmu = data;
39 	u32 status;
40 
41 	status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS);
42 	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
43 
44 	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
45 		dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
46 
47 		a6xx_gmu_fault(gmu);
48 	}
49 
50 	if (status &  A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
51 		dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n");
52 
53 	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
54 		dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n",
55 			gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS));
56 
57 	return IRQ_HANDLED;
58 }
59 
60 static irqreturn_t a6xx_hfi_irq(int irq, void *data)
61 {
62 	struct a6xx_gmu *gmu = data;
63 	u32 status;
64 
65 	status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
66 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
67 
68 	if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
69 		dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
70 
71 		a6xx_gmu_fault(gmu);
72 	}
73 
74 	return IRQ_HANDLED;
75 }
76 
77 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu)
78 {
79 	u32 val;
80 
81 	/* This can be called from gpu state code so make sure GMU is valid */
82 	if (!gmu->initialized)
83 		return false;
84 
85 	val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
86 
87 	return !(val &
88 		(A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF |
89 		A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF));
90 }
91 
92 /* Check to see if the GX rail is still powered */
93 bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
94 {
95 	u32 val;
96 
97 	/* This can be called from gpu state code so make sure GMU is valid */
98 	if (!gmu->initialized)
99 		return false;
100 
101 	val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
102 
103 	return !(val &
104 		(A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
105 		A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
106 }
107 
108 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
109 		       bool suspended)
110 {
111 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
112 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
113 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
114 	u32 perf_index;
115 	unsigned long gpu_freq;
116 	int ret = 0;
117 
118 	gpu_freq = dev_pm_opp_get_freq(opp);
119 
120 	if (gpu_freq == gmu->freq)
121 		return;
122 
123 	for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++)
124 		if (gpu_freq == gmu->gpu_freqs[perf_index])
125 			break;
126 
127 	gmu->current_perf_index = perf_index;
128 	gmu->freq = gmu->gpu_freqs[perf_index];
129 
130 	trace_msm_gmu_freq_change(gmu->freq, perf_index);
131 
132 	/*
133 	 * This can get called from devfreq while the hardware is idle. Don't
134 	 * bring up the power if it isn't already active. All we're doing here
135 	 * is updating the frequency so that when we come back online we're at
136 	 * the right rate.
137 	 */
138 	if (suspended)
139 		return;
140 
141 	if (!gmu->legacy) {
142 		a6xx_hfi_set_freq(gmu, perf_index);
143 		dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
144 		return;
145 	}
146 
147 	gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
148 
149 	gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
150 			((3 & 0xf) << 28) | perf_index);
151 
152 	/*
153 	 * Send an invalid index as a vote for the bus bandwidth and let the
154 	 * firmware decide on the right vote
155 	 */
156 	gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff);
157 
158 	/* Set and clear the OOB for DCVS to trigger the GMU */
159 	a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
160 	a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
161 
162 	ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
163 	if (ret)
164 		dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
165 
166 	dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
167 }
168 
169 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
170 {
171 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
172 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
173 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
174 
175 	return  gmu->freq;
176 }
177 
178 static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
179 {
180 	u32 val;
181 	int local = gmu->idle_level;
182 
183 	/* SPTP and IFPC both report as IFPC */
184 	if (gmu->idle_level == GMU_IDLE_STATE_SPTP)
185 		local = GMU_IDLE_STATE_IFPC;
186 
187 	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
188 
189 	if (val == local) {
190 		if (gmu->idle_level != GMU_IDLE_STATE_IFPC ||
191 			!a6xx_gmu_gx_is_on(gmu))
192 			return true;
193 	}
194 
195 	return false;
196 }
197 
198 /* Wait for the GMU to get to its most idle state */
199 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu)
200 {
201 	return spin_until(a6xx_gmu_check_idle_level(gmu));
202 }
203 
204 static int a6xx_gmu_start(struct a6xx_gmu *gmu)
205 {
206 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
207 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
208 	u32 mask, reset_val, val;
209 	int ret;
210 
211 	val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8);
212 	if (val <= 0x20010004) {
213 		mask = 0xffffffff;
214 		reset_val = 0xbabeface;
215 	} else {
216 		mask = 0x1ff;
217 		reset_val = 0x100;
218 	}
219 
220 	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
221 
222 	/* Set the log wptr index
223 	 * note: downstream saves the value in poweroff and restores it here
224 	 */
225 	if (adreno_is_a7xx(adreno_gpu))
226 		gmu_write(gmu, REG_A7XX_GMU_GENERAL_9, 0);
227 	else
228 		gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0);
229 
230 
231 	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
232 
233 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
234 		(val & mask) == reset_val, 100, 10000);
235 
236 	if (ret)
237 		DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n");
238 
239 	return ret;
240 }
241 
242 static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu)
243 {
244 	u32 val;
245 	int ret;
246 
247 	gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
248 
249 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
250 		val & 1, 100, 10000);
251 	if (ret)
252 		DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n");
253 
254 	return ret;
255 }
256 
257 struct a6xx_gmu_oob_bits {
258 	int set, ack, set_new, ack_new, clear, clear_new;
259 	const char *name;
260 };
261 
262 /* These are the interrupt / ack bits for each OOB request that are set
263  * in a6xx_gmu_set_oob and a6xx_clear_oob
264  */
265 static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = {
266 	[GMU_OOB_GPU_SET] = {
267 		.name = "GPU_SET",
268 		.set = 16,
269 		.ack = 24,
270 		.set_new = 30,
271 		.ack_new = 31,
272 		.clear = 24,
273 		.clear_new = 31,
274 	},
275 
276 	[GMU_OOB_PERFCOUNTER_SET] = {
277 		.name = "PERFCOUNTER",
278 		.set = 17,
279 		.ack = 25,
280 		.set_new = 28,
281 		.ack_new = 30,
282 		.clear = 25,
283 		.clear_new = 29,
284 	},
285 
286 	[GMU_OOB_BOOT_SLUMBER] = {
287 		.name = "BOOT_SLUMBER",
288 		.set = 22,
289 		.ack = 30,
290 		.clear = 30,
291 	},
292 
293 	[GMU_OOB_DCVS_SET] = {
294 		.name = "GPU_DCVS",
295 		.set = 23,
296 		.ack = 31,
297 		.clear = 31,
298 	},
299 };
300 
301 /* Trigger a OOB (out of band) request to the GMU */
302 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
303 {
304 	int ret;
305 	u32 val;
306 	int request, ack;
307 
308 	WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
309 
310 	if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
311 		return -EINVAL;
312 
313 	if (gmu->legacy) {
314 		request = a6xx_gmu_oob_bits[state].set;
315 		ack = a6xx_gmu_oob_bits[state].ack;
316 	} else {
317 		request = a6xx_gmu_oob_bits[state].set_new;
318 		ack = a6xx_gmu_oob_bits[state].ack_new;
319 		if (!request || !ack) {
320 			DRM_DEV_ERROR(gmu->dev,
321 				      "Invalid non-legacy GMU request %s\n",
322 				      a6xx_gmu_oob_bits[state].name);
323 			return -EINVAL;
324 		}
325 	}
326 
327 	/* Trigger the equested OOB operation */
328 	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request);
329 
330 	/* Wait for the acknowledge interrupt */
331 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
332 		val & (1 << ack), 100, 10000);
333 
334 	if (ret)
335 		DRM_DEV_ERROR(gmu->dev,
336 			"Timeout waiting for GMU OOB set %s: 0x%x\n",
337 				a6xx_gmu_oob_bits[state].name,
338 				gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
339 
340 	/* Clear the acknowledge interrupt */
341 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack);
342 
343 	return ret;
344 }
345 
346 /* Clear a pending OOB state in the GMU */
347 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
348 {
349 	int bit;
350 
351 	WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
352 
353 	if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
354 		return;
355 
356 	if (gmu->legacy)
357 		bit = a6xx_gmu_oob_bits[state].clear;
358 	else
359 		bit = a6xx_gmu_oob_bits[state].clear_new;
360 
361 	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit);
362 }
363 
364 /* Enable CPU control of SPTP power power collapse */
365 int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
366 {
367 	int ret;
368 	u32 val;
369 
370 	if (!gmu->legacy)
371 		return 0;
372 
373 	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
374 
375 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
376 		(val & 0x38) == 0x28, 1, 100);
377 
378 	if (ret) {
379 		DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
380 			gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
381 	}
382 
383 	return 0;
384 }
385 
386 /* Disable CPU control of SPTP power power collapse */
387 void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
388 {
389 	u32 val;
390 	int ret;
391 
392 	if (!gmu->legacy)
393 		return;
394 
395 	/* Make sure retention is on */
396 	gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
397 
398 	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001);
399 
400 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
401 		(val & 0x04), 100, 10000);
402 
403 	if (ret)
404 		DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
405 			gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
406 }
407 
408 /* Let the GMU know we are starting a boot sequence */
409 static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu)
410 {
411 	u32 vote;
412 
413 	/* Let the GMU know we are getting ready for boot */
414 	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0);
415 
416 	/* Choose the "default" power level as the highest available */
417 	vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1];
418 
419 	gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff);
420 	gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff);
421 
422 	/* Let the GMU know the boot sequence has started */
423 	return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
424 }
425 
426 /* Let the GMU know that we are about to go into slumber */
427 static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
428 {
429 	int ret;
430 
431 	/* Disable the power counter so the GMU isn't busy */
432 	gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
433 
434 	/* Disable SPTP_PC if the CPU is responsible for it */
435 	if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
436 		a6xx_sptprac_disable(gmu);
437 
438 	if (!gmu->legacy) {
439 		ret = a6xx_hfi_send_prep_slumber(gmu);
440 		goto out;
441 	}
442 
443 	/* Tell the GMU to get ready to slumber */
444 	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
445 
446 	ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
447 	a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
448 
449 	if (!ret) {
450 		/* Check to see if the GMU really did slumber */
451 		if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
452 			!= 0x0f) {
453 			DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n");
454 			ret = -ETIMEDOUT;
455 		}
456 	}
457 
458 out:
459 	/* Put fence into allow mode */
460 	gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
461 	return ret;
462 }
463 
464 static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
465 {
466 	int ret;
467 	u32 val;
468 
469 	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, BIT(1));
470 
471 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
472 		val & (1 << 1), 100, 10000);
473 	if (ret) {
474 		DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n");
475 		return ret;
476 	}
477 
478 	ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
479 		!val, 100, 10000);
480 
481 	if (ret) {
482 		DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
483 		return ret;
484 	}
485 
486 	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
487 
488 	return 0;
489 }
490 
491 static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
492 {
493 	int ret;
494 	u32 val;
495 
496 	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
497 
498 	ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
499 		val, val & (1 << 16), 100, 10000);
500 	if (ret)
501 		DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n");
502 
503 	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
504 }
505 
506 static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value)
507 {
508 	writel(value, ptr + (offset << 2));
509 }
510 
511 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
512 		const char *name);
513 
514 static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
515 {
516 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
517 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
518 	struct platform_device *pdev = to_platform_device(gmu->dev);
519 	void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
520 	u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0;
521 	void __iomem *seqptr = NULL;
522 	uint32_t pdc_address_offset;
523 	bool pdc_in_aop = false;
524 
525 	if (IS_ERR(pdcptr))
526 		goto err;
527 
528 	if (adreno_is_a650(adreno_gpu) ||
529 	    adreno_is_a660_family(adreno_gpu) ||
530 	    adreno_is_a7xx(adreno_gpu))
531 		pdc_in_aop = true;
532 	else if (adreno_is_a618(adreno_gpu) || adreno_is_a640_family(adreno_gpu))
533 		pdc_address_offset = 0x30090;
534 	else if (adreno_is_a619(adreno_gpu))
535 		pdc_address_offset = 0x300a0;
536 	else
537 		pdc_address_offset = 0x30080;
538 
539 	if (!pdc_in_aop) {
540 		seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq");
541 		if (IS_ERR(seqptr))
542 			goto err;
543 	}
544 
545 	/* Disable SDE clock gating */
546 	gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
547 
548 	/* Setup RSC PDC handshake for sleep and wakeup */
549 	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
550 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
551 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
552 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
553 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
554 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4,
555 		       adreno_is_a740_family(adreno_gpu) ? 0x80000021 : 0x80000000);
556 	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
557 	gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
558 	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
559 	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
560 	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
561 
562 	/* The second spin of A7xx GPUs messed with some register offsets.. */
563 	if (adreno_is_a740_family(adreno_gpu))
564 		seqmem0_drv0_reg = REG_A7XX_RSCC_SEQ_MEM_0_DRV0_A740;
565 
566 	/* Load RSC sequencer uCode for sleep and wakeup */
567 	if (adreno_is_a650_family(adreno_gpu) ||
568 	    adreno_is_a7xx(adreno_gpu)) {
569 		gmu_write_rscc(gmu, seqmem0_drv0_reg, 0xeaaae5a0);
570 		gmu_write_rscc(gmu, seqmem0_drv0_reg + 1, 0xe1a1ebab);
571 		gmu_write_rscc(gmu, seqmem0_drv0_reg + 2, 0xa2e0a581);
572 		gmu_write_rscc(gmu, seqmem0_drv0_reg + 3, 0xecac82e2);
573 		gmu_write_rscc(gmu, seqmem0_drv0_reg + 4, 0x0020edad);
574 	} else {
575 		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
576 		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
577 		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
578 		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
579 		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
580 	}
581 
582 	if (pdc_in_aop)
583 		goto setup_pdc;
584 
585 	/* Load PDC sequencer uCode for power up and power down sequence */
586 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
587 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
588 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
589 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
590 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
591 
592 	/* Set TCS commands used by PDC sequence for low power modes */
593 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
594 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
595 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
596 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
597 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
598 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
599 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
600 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
601 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
602 
603 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
604 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset);
605 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
606 
607 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
608 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
609 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
610 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
611 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
612 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
613 
614 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
615 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
616 	if (adreno_is_a618(adreno_gpu) || adreno_is_a619(adreno_gpu) ||
617 			adreno_is_a650_family(adreno_gpu))
618 		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2);
619 	else
620 		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
621 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
622 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset);
623 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
624 
625 	/* Setup GPU PDC */
626 setup_pdc:
627 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
628 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
629 
630 	/* ensure no writes happen before the uCode is fully written */
631 	wmb();
632 
633 	a6xx_rpmh_stop(gmu);
634 
635 err:
636 	if (!IS_ERR_OR_NULL(pdcptr))
637 		iounmap(pdcptr);
638 	if (!IS_ERR_OR_NULL(seqptr))
639 		iounmap(seqptr);
640 }
641 
642 /*
643  * The lowest 16 bits of this value are the number of XO clock cycles for main
644  * hysteresis which is set at 0x1680 cycles (300 us).  The higher 16 bits are
645  * for the shorter hysteresis that happens after main - this is 0xa (.5 us)
646  */
647 
648 #define GMU_PWR_COL_HYST 0x000a1680
649 
650 /* Set up the idle state for the GMU */
651 static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
652 {
653 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
654 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
655 
656 	/* Disable GMU WB/RB buffer */
657 	gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
658 	gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1);
659 	gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1);
660 
661 	/* A7xx knows better by default! */
662 	if (adreno_is_a7xx(adreno_gpu))
663 		return;
664 
665 	gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
666 
667 	switch (gmu->idle_level) {
668 	case GMU_IDLE_STATE_IFPC:
669 		gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
670 			GMU_PWR_COL_HYST);
671 		gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
672 			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
673 			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
674 		fallthrough;
675 	case GMU_IDLE_STATE_SPTP:
676 		gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
677 			GMU_PWR_COL_HYST);
678 		gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
679 			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
680 			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE);
681 	}
682 
683 	/* Enable RPMh GPU client */
684 	gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0,
685 		A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE |
686 		A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE |
687 		A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE |
688 		A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE |
689 		A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE |
690 		A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
691 }
692 
693 struct block_header {
694 	u32 addr;
695 	u32 size;
696 	u32 type;
697 	u32 value;
698 	u32 data[];
699 };
700 
701 static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk)
702 {
703 	if (!in_range(blk->addr, bo->iova, bo->size))
704 		return false;
705 
706 	memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size);
707 	return true;
708 }
709 
710 static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
711 {
712 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
713 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
714 	const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU];
715 	const struct block_header *blk;
716 	u32 reg_offset;
717 
718 	u32 itcm_base = 0x00000000;
719 	u32 dtcm_base = 0x00040000;
720 
721 	if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu))
722 		dtcm_base = 0x10004000;
723 
724 	if (gmu->legacy) {
725 		/* Sanity check the size of the firmware that was loaded */
726 		if (fw_image->size > 0x8000) {
727 			DRM_DEV_ERROR(gmu->dev,
728 				"GMU firmware is bigger than the available region\n");
729 			return -EINVAL;
730 		}
731 
732 		gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START,
733 			       (u32*) fw_image->data, fw_image->size);
734 		return 0;
735 	}
736 
737 
738 	for (blk = (const struct block_header *) fw_image->data;
739 	     (const u8*) blk < fw_image->data + fw_image->size;
740 	     blk = (const struct block_header *) &blk->data[blk->size >> 2]) {
741 		if (blk->size == 0)
742 			continue;
743 
744 		if (in_range(blk->addr, itcm_base, SZ_16K)) {
745 			reg_offset = (blk->addr - itcm_base) >> 2;
746 			gmu_write_bulk(gmu,
747 				REG_A6XX_GMU_CM3_ITCM_START + reg_offset,
748 				blk->data, blk->size);
749 		} else if (in_range(blk->addr, dtcm_base, SZ_16K)) {
750 			reg_offset = (blk->addr - dtcm_base) >> 2;
751 			gmu_write_bulk(gmu,
752 				REG_A6XX_GMU_CM3_DTCM_START + reg_offset,
753 				blk->data, blk->size);
754 		} else if (!fw_block_mem(&gmu->icache, blk) &&
755 			   !fw_block_mem(&gmu->dcache, blk) &&
756 			   !fw_block_mem(&gmu->dummy, blk)) {
757 			DRM_DEV_ERROR(gmu->dev,
758 				"failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n",
759 				blk->addr, blk->size, blk->data[0]);
760 		}
761 	}
762 
763 	return 0;
764 }
765 
766 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
767 {
768 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
769 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
770 	const struct a6xx_info *a6xx_info = adreno_gpu->info->a6xx;
771 	u32 fence_range_lower, fence_range_upper;
772 	u32 chipid = 0;
773 	int ret;
774 
775 	/* Vote veto for FAL10 */
776 	if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) {
777 		gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1);
778 		gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
779 	}
780 
781 	/* Turn on TCM (Tightly Coupled Memory) retention */
782 	if (adreno_is_a7xx(adreno_gpu))
783 		a6xx_llc_write(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL, 1);
784 	else
785 		gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
786 
787 	if (state == GMU_WARM_BOOT) {
788 		ret = a6xx_rpmh_start(gmu);
789 		if (ret)
790 			return ret;
791 	} else {
792 		if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU],
793 			"GMU firmware is not loaded\n"))
794 			return -ENOENT;
795 
796 		ret = a6xx_rpmh_start(gmu);
797 		if (ret)
798 			return ret;
799 
800 		ret = a6xx_gmu_fw_load(gmu);
801 		if (ret)
802 			return ret;
803 	}
804 
805 	/* Clear init result to make sure we are getting a fresh value */
806 	gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
807 	gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
808 
809 	/* Write the iova of the HFI table */
810 	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova);
811 	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
812 
813 	if (adreno_is_a7xx(adreno_gpu)) {
814 		fence_range_upper = 0x32;
815 		fence_range_lower = 0x8a0;
816 	} else {
817 		fence_range_upper = 0xa;
818 		fence_range_lower = 0xa0;
819 	}
820 
821 	gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
822 		  BIT(31) |
823 		  FIELD_PREP(GENMASK(30, 18), fence_range_upper) |
824 		  FIELD_PREP(GENMASK(17, 0), fence_range_lower));
825 
826 	/*
827 	 * Snapshots toggle the NMI bit which will result in a jump to the NMI
828 	 * handler instead of __main. Set the M3 config value to avoid that.
829 	 */
830 	gmu_write(gmu, REG_A6XX_GMU_CM3_CFG, 0x4052);
831 
832 	if (a6xx_info->gmu_chipid) {
833 		chipid = a6xx_info->gmu_chipid;
834 	} else {
835 		/*
836 		 * Note that the GMU has a slightly different layout for
837 		 * chip_id, for whatever reason, so a bit of massaging
838 		 * is needed.  The upper 16b are the same, but minor and
839 		 * patchid are packed in four bits each with the lower
840 		 * 8b unused:
841 		 */
842 		chipid  = adreno_gpu->chip_id & 0xffff0000;
843 		chipid |= (adreno_gpu->chip_id << 4) & 0xf000; /* minor */
844 		chipid |= (adreno_gpu->chip_id << 8) & 0x0f00; /* patchid */
845 	}
846 
847 	if (adreno_is_a7xx(adreno_gpu)) {
848 		gmu_write(gmu, REG_A7XX_GMU_GENERAL_10, chipid);
849 		gmu_write(gmu, REG_A7XX_GMU_GENERAL_8,
850 			  (gmu->log.iova & GENMASK(31, 12)) |
851 			  ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0)));
852 	} else {
853 		gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
854 
855 		gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
856 			  gmu->log.iova | (gmu->log.size / SZ_4K - 1));
857 	}
858 
859 	/* Set up the lowest idle level on the GMU */
860 	a6xx_gmu_power_config(gmu);
861 
862 	ret = a6xx_gmu_start(gmu);
863 	if (ret)
864 		return ret;
865 
866 	if (gmu->legacy) {
867 		ret = a6xx_gmu_gfx_rail_on(gmu);
868 		if (ret)
869 			return ret;
870 	}
871 
872 	/* Enable SPTP_PC if the CPU is responsible for it */
873 	if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
874 		ret = a6xx_sptprac_enable(gmu);
875 		if (ret)
876 			return ret;
877 	}
878 
879 	ret = a6xx_gmu_hfi_start(gmu);
880 	if (ret)
881 		return ret;
882 
883 	/* FIXME: Do we need this wmb() here? */
884 	wmb();
885 
886 	return 0;
887 }
888 
889 #define A6XX_HFI_IRQ_MASK \
890 	(A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
891 
892 #define A6XX_GMU_IRQ_MASK \
893 	(A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \
894 	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
895 	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
896 
897 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
898 {
899 	disable_irq(gmu->gmu_irq);
900 	disable_irq(gmu->hfi_irq);
901 
902 	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0);
903 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
904 }
905 
906 static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
907 {
908 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
909 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
910 	u32 val, seqmem_off = 0;
911 
912 	/* The second spin of A7xx GPUs messed with some register offsets.. */
913 	if (adreno_is_a740_family(adreno_gpu))
914 		seqmem_off = 4;
915 
916 	/* Make sure there are no outstanding RPMh votes */
917 	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS + seqmem_off,
918 		val, (val & 1), 100, 10000);
919 	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS + seqmem_off,
920 		val, (val & 1), 100, 10000);
921 	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS + seqmem_off,
922 		val, (val & 1), 100, 10000);
923 	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS + seqmem_off,
924 		val, (val & 1), 100, 1000);
925 }
926 
927 /* Force the GMU off in case it isn't responsive */
928 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
929 {
930 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
931 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
932 	struct msm_gpu *gpu = &adreno_gpu->base;
933 
934 	/*
935 	 * Turn off keep alive that might have been enabled by the hang
936 	 * interrupt
937 	 */
938 	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
939 
940 	/* Flush all the queues */
941 	a6xx_hfi_stop(gmu);
942 
943 	/* Stop the interrupts */
944 	a6xx_gmu_irq_disable(gmu);
945 
946 	/* Force off SPTP in case the GMU is managing it */
947 	a6xx_sptprac_disable(gmu);
948 
949 	/* Make sure there are no outstanding RPMh votes */
950 	a6xx_gmu_rpmh_off(gmu);
951 
952 	/* Clear the WRITEDROPPED fields and put fence into allow mode */
953 	gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7);
954 	gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
955 
956 	/* Make sure the above writes go through */
957 	wmb();
958 
959 	/* Halt the gmu cm3 core */
960 	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
961 
962 	a6xx_bus_clear_pending_transactions(adreno_gpu, true);
963 
964 	/* Reset GPU core blocks */
965 	a6xx_gpu_sw_reset(gpu, true);
966 }
967 
968 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
969 {
970 	struct dev_pm_opp *gpu_opp;
971 	unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
972 
973 	gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
974 	if (IS_ERR(gpu_opp))
975 		return;
976 
977 	gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */
978 	a6xx_gmu_set_freq(gpu, gpu_opp, false);
979 	dev_pm_opp_put(gpu_opp);
980 }
981 
982 static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
983 {
984 	struct dev_pm_opp *gpu_opp;
985 	unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
986 
987 	gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
988 	if (IS_ERR(gpu_opp))
989 		return;
990 
991 	dev_pm_opp_set_opp(&gpu->pdev->dev, gpu_opp);
992 	dev_pm_opp_put(gpu_opp);
993 }
994 
995 int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
996 {
997 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
998 	struct msm_gpu *gpu = &adreno_gpu->base;
999 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1000 	int status, ret;
1001 
1002 	if (WARN(!gmu->initialized, "The GMU is not set up yet\n"))
1003 		return -EINVAL;
1004 
1005 	gmu->hung = false;
1006 
1007 	/* Notify AOSS about the ACD state (unimplemented for now => disable it) */
1008 	if (!IS_ERR(gmu->qmp)) {
1009 		ret = qmp_send(gmu->qmp, "{class: gpu, res: acd, val: %d}",
1010 			       0 /* Hardcode ACD to be disabled for now */);
1011 		if (ret)
1012 			dev_err(gmu->dev, "failed to send GPU ACD state\n");
1013 	}
1014 
1015 	/* Turn on the resources */
1016 	pm_runtime_get_sync(gmu->dev);
1017 
1018 	/*
1019 	 * "enable" the GX power domain which won't actually do anything but it
1020 	 * will make sure that the refcounting is correct in case we need to
1021 	 * bring down the GX after a GMU failure
1022 	 */
1023 	if (!IS_ERR_OR_NULL(gmu->gxpd))
1024 		pm_runtime_get_sync(gmu->gxpd);
1025 
1026 	/* Use a known rate to bring up the GMU */
1027 	clk_set_rate(gmu->core_clk, 200000000);
1028 	clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ?
1029 		     200000000 : 150000000);
1030 	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
1031 	if (ret) {
1032 		pm_runtime_put(gmu->gxpd);
1033 		pm_runtime_put(gmu->dev);
1034 		return ret;
1035 	}
1036 
1037 	/* Set the bus quota to a reasonable value for boot */
1038 	a6xx_gmu_set_initial_bw(gpu, gmu);
1039 
1040 	/* Enable the GMU interrupt */
1041 	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
1042 	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK);
1043 	enable_irq(gmu->gmu_irq);
1044 
1045 	/* Check to see if we are doing a cold or warm boot */
1046 	if (adreno_is_a7xx(adreno_gpu)) {
1047 		status = a6xx_llc_read(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL) == 1 ?
1048 			GMU_WARM_BOOT : GMU_COLD_BOOT;
1049 	} else if (gmu->legacy) {
1050 		status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
1051 			GMU_WARM_BOOT : GMU_COLD_BOOT;
1052 	} else {
1053 		/*
1054 		 * Warm boot path does not work on newer A6xx GPUs
1055 		 * Presumably this is because icache/dcache regions must be restored
1056 		 */
1057 		status = GMU_COLD_BOOT;
1058 	}
1059 
1060 	ret = a6xx_gmu_fw_start(gmu, status);
1061 	if (ret)
1062 		goto out;
1063 
1064 	ret = a6xx_hfi_start(gmu, status);
1065 	if (ret)
1066 		goto out;
1067 
1068 	/*
1069 	 * Turn on the GMU firmware fault interrupt after we know the boot
1070 	 * sequence is successful
1071 	 */
1072 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
1073 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK);
1074 	enable_irq(gmu->hfi_irq);
1075 
1076 	/* Set the GPU to the current freq */
1077 	a6xx_gmu_set_initial_freq(gpu, gmu);
1078 
1079 out:
1080 	/* On failure, shut down the GMU to leave it in a good state */
1081 	if (ret) {
1082 		disable_irq(gmu->gmu_irq);
1083 		a6xx_rpmh_stop(gmu);
1084 		pm_runtime_put(gmu->gxpd);
1085 		pm_runtime_put(gmu->dev);
1086 	}
1087 
1088 	return ret;
1089 }
1090 
1091 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
1092 {
1093 	u32 reg;
1094 
1095 	if (!gmu->initialized)
1096 		return true;
1097 
1098 	reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
1099 
1100 	if (reg &  A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB)
1101 		return false;
1102 
1103 	return true;
1104 }
1105 
1106 /* Gracefully try to shut down the GMU and by extension the GPU */
1107 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
1108 {
1109 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1110 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1111 	u32 val;
1112 
1113 	/*
1114 	 * The GMU may still be in slumber unless the GPU started so check and
1115 	 * skip putting it back into slumber if so
1116 	 */
1117 	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
1118 
1119 	if (val != 0xf) {
1120 		int ret = a6xx_gmu_wait_for_idle(gmu);
1121 
1122 		/* If the GMU isn't responding assume it is hung */
1123 		if (ret) {
1124 			a6xx_gmu_force_off(gmu);
1125 			return;
1126 		}
1127 
1128 		a6xx_bus_clear_pending_transactions(adreno_gpu, a6xx_gpu->hung);
1129 
1130 		/* tell the GMU we want to slumber */
1131 		ret = a6xx_gmu_notify_slumber(gmu);
1132 		if (ret) {
1133 			a6xx_gmu_force_off(gmu);
1134 			return;
1135 		}
1136 
1137 		ret = gmu_poll_timeout(gmu,
1138 			REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
1139 			!(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB),
1140 			100, 10000);
1141 
1142 		/*
1143 		 * Let the user know we failed to slumber but don't worry too
1144 		 * much because we are powering down anyway
1145 		 */
1146 
1147 		if (ret)
1148 			DRM_DEV_ERROR(gmu->dev,
1149 				"Unable to slumber GMU: status = 0%x/0%x\n",
1150 				gmu_read(gmu,
1151 					REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
1152 				gmu_read(gmu,
1153 					REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2));
1154 	}
1155 
1156 	/* Turn off HFI */
1157 	a6xx_hfi_stop(gmu);
1158 
1159 	/* Stop the interrupts and mask the hardware */
1160 	a6xx_gmu_irq_disable(gmu);
1161 
1162 	/* Tell RPMh to power off the GPU */
1163 	a6xx_rpmh_stop(gmu);
1164 }
1165 
1166 
1167 int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
1168 {
1169 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1170 	struct msm_gpu *gpu = &a6xx_gpu->base.base;
1171 
1172 	if (!pm_runtime_active(gmu->dev))
1173 		return 0;
1174 
1175 	/*
1176 	 * Force the GMU off if we detected a hang, otherwise try to shut it
1177 	 * down gracefully
1178 	 */
1179 	if (gmu->hung)
1180 		a6xx_gmu_force_off(gmu);
1181 	else
1182 		a6xx_gmu_shutdown(gmu);
1183 
1184 	/* Remove the bus vote */
1185 	dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
1186 
1187 	/*
1188 	 * Make sure the GX domain is off before turning off the GMU (CX)
1189 	 * domain. Usually the GMU does this but only if the shutdown sequence
1190 	 * was successful
1191 	 */
1192 	if (!IS_ERR_OR_NULL(gmu->gxpd))
1193 		pm_runtime_put_sync(gmu->gxpd);
1194 
1195 	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
1196 
1197 	pm_runtime_put_sync(gmu->dev);
1198 
1199 	return 0;
1200 }
1201 
1202 static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu)
1203 {
1204 	msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace);
1205 	msm_gem_kernel_put(gmu->debug.obj, gmu->aspace);
1206 	msm_gem_kernel_put(gmu->icache.obj, gmu->aspace);
1207 	msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace);
1208 	msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace);
1209 	msm_gem_kernel_put(gmu->log.obj, gmu->aspace);
1210 
1211 	gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu);
1212 	msm_gem_address_space_put(gmu->aspace);
1213 }
1214 
1215 static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo,
1216 		size_t size, u64 iova, const char *name)
1217 {
1218 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1219 	struct drm_device *dev = a6xx_gpu->base.base.dev;
1220 	uint32_t flags = MSM_BO_WC;
1221 	u64 range_start, range_end;
1222 	int ret;
1223 
1224 	size = PAGE_ALIGN(size);
1225 	if (!iova) {
1226 		/* no fixed address - use GMU's uncached range */
1227 		range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */
1228 		range_end = 0x80000000;
1229 	} else {
1230 		/* range for fixed address */
1231 		range_start = iova;
1232 		range_end = iova + size;
1233 		/* use IOMMU_PRIV for icache/dcache */
1234 		flags |= MSM_BO_MAP_PRIV;
1235 	}
1236 
1237 	bo->obj = msm_gem_new(dev, size, flags);
1238 	if (IS_ERR(bo->obj))
1239 		return PTR_ERR(bo->obj);
1240 
1241 	ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova,
1242 					     range_start, range_end);
1243 	if (ret) {
1244 		drm_gem_object_put(bo->obj);
1245 		return ret;
1246 	}
1247 
1248 	bo->virt = msm_gem_get_vaddr(bo->obj);
1249 	bo->size = size;
1250 
1251 	msm_gem_object_set_name(bo->obj, name);
1252 
1253 	return 0;
1254 }
1255 
1256 static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
1257 {
1258 	struct msm_mmu *mmu;
1259 
1260 	mmu = msm_iommu_new(gmu->dev, 0);
1261 	if (!mmu)
1262 		return -ENODEV;
1263 	if (IS_ERR(mmu))
1264 		return PTR_ERR(mmu);
1265 
1266 	gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x80000000);
1267 	if (IS_ERR(gmu->aspace))
1268 		return PTR_ERR(gmu->aspace);
1269 
1270 	return 0;
1271 }
1272 
1273 /* Return the 'arc-level' for the given frequency */
1274 static unsigned int a6xx_gmu_get_arc_level(struct device *dev,
1275 					   unsigned long freq)
1276 {
1277 	struct dev_pm_opp *opp;
1278 	unsigned int val;
1279 
1280 	if (!freq)
1281 		return 0;
1282 
1283 	opp = dev_pm_opp_find_freq_exact(dev, freq, true);
1284 	if (IS_ERR(opp))
1285 		return 0;
1286 
1287 	val = dev_pm_opp_get_level(opp);
1288 
1289 	dev_pm_opp_put(opp);
1290 
1291 	return val;
1292 }
1293 
1294 static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
1295 		unsigned long *freqs, int freqs_count, const char *id)
1296 {
1297 	int i, j;
1298 	const u16 *pri, *sec;
1299 	size_t pri_count, sec_count;
1300 
1301 	pri = cmd_db_read_aux_data(id, &pri_count);
1302 	if (IS_ERR(pri))
1303 		return PTR_ERR(pri);
1304 	/*
1305 	 * The data comes back as an array of unsigned shorts so adjust the
1306 	 * count accordingly
1307 	 */
1308 	pri_count >>= 1;
1309 	if (!pri_count)
1310 		return -EINVAL;
1311 
1312 	/*
1313 	 * Some targets have a separate gfx mxc rail. So try to read that first and then fall back
1314 	 * to regular mx rail if it is missing
1315 	 */
1316 	sec = cmd_db_read_aux_data("gmxc.lvl", &sec_count);
1317 	if (IS_ERR(sec) && sec != ERR_PTR(-EPROBE_DEFER))
1318 		sec = cmd_db_read_aux_data("mx.lvl", &sec_count);
1319 	if (IS_ERR(sec))
1320 		return PTR_ERR(sec);
1321 
1322 	sec_count >>= 1;
1323 	if (!sec_count)
1324 		return -EINVAL;
1325 
1326 	/* Construct a vote for each frequency */
1327 	for (i = 0; i < freqs_count; i++) {
1328 		u8 pindex = 0, sindex = 0;
1329 		unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
1330 
1331 		/* Get the primary index that matches the arc level */
1332 		for (j = 0; j < pri_count; j++) {
1333 			if (pri[j] >= level) {
1334 				pindex = j;
1335 				break;
1336 			}
1337 		}
1338 
1339 		if (j == pri_count) {
1340 			DRM_DEV_ERROR(dev,
1341 				      "Level %u not found in the RPMh list\n",
1342 				      level);
1343 			DRM_DEV_ERROR(dev, "Available levels:\n");
1344 			for (j = 0; j < pri_count; j++)
1345 				DRM_DEV_ERROR(dev, "  %u\n", pri[j]);
1346 
1347 			return -EINVAL;
1348 		}
1349 
1350 		/*
1351 		 * Look for a level in in the secondary list that matches. If
1352 		 * nothing fits, use the maximum non zero vote
1353 		 */
1354 
1355 		for (j = 0; j < sec_count; j++) {
1356 			if (sec[j] >= level) {
1357 				sindex = j;
1358 				break;
1359 			} else if (sec[j]) {
1360 				sindex = j;
1361 			}
1362 		}
1363 
1364 		/* Construct the vote */
1365 		votes[i] = ((pri[pindex] & 0xffff) << 16) |
1366 			(sindex << 8) | pindex;
1367 	}
1368 
1369 	return 0;
1370 }
1371 
1372 /*
1373  * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
1374  * to construct the list of votes on the CPU and send it over. Query the RPMh
1375  * voltage levels and build the votes
1376  */
1377 
1378 static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu)
1379 {
1380 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1381 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1382 	struct msm_gpu *gpu = &adreno_gpu->base;
1383 	int ret;
1384 
1385 	/* Build the GX votes */
1386 	ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
1387 		gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl");
1388 
1389 	/* Build the CX votes */
1390 	ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
1391 		gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl");
1392 
1393 	return ret;
1394 }
1395 
1396 static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs,
1397 		u32 size)
1398 {
1399 	int count = dev_pm_opp_get_opp_count(dev);
1400 	struct dev_pm_opp *opp;
1401 	int i, index = 0;
1402 	unsigned long freq = 1;
1403 
1404 	/*
1405 	 * The OPP table doesn't contain the "off" frequency level so we need to
1406 	 * add 1 to the table size to account for it
1407 	 */
1408 
1409 	if (WARN(count + 1 > size,
1410 		"The GMU frequency table is being truncated\n"))
1411 		count = size - 1;
1412 
1413 	/* Set the "off" frequency */
1414 	freqs[index++] = 0;
1415 
1416 	for (i = 0; i < count; i++) {
1417 		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
1418 		if (IS_ERR(opp))
1419 			break;
1420 
1421 		dev_pm_opp_put(opp);
1422 		freqs[index++] = freq++;
1423 	}
1424 
1425 	return index;
1426 }
1427 
1428 static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu)
1429 {
1430 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1431 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1432 	struct msm_gpu *gpu = &adreno_gpu->base;
1433 
1434 	int ret = 0;
1435 
1436 	/*
1437 	 * The GMU handles its own frequency switching so build a list of
1438 	 * available frequencies to send during initialization
1439 	 */
1440 	ret = devm_pm_opp_of_add_table(gmu->dev);
1441 	if (ret) {
1442 		DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n");
1443 		return ret;
1444 	}
1445 
1446 	gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev,
1447 		gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs));
1448 
1449 	/*
1450 	 * The GMU also handles GPU frequency switching so build a list
1451 	 * from the GPU OPP table
1452 	 */
1453 	gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
1454 		gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
1455 
1456 	gmu->current_perf_index = gmu->nr_gpu_freqs - 1;
1457 
1458 	/* Build the list of RPMh votes that we'll send to the GMU */
1459 	return a6xx_gmu_rpmh_votes_init(gmu);
1460 }
1461 
1462 static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
1463 {
1464 	int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks);
1465 
1466 	if (ret < 1)
1467 		return ret;
1468 
1469 	gmu->nr_clocks = ret;
1470 
1471 	gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks,
1472 		gmu->nr_clocks, "gmu");
1473 
1474 	gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks,
1475 		gmu->nr_clocks, "hub");
1476 
1477 	return 0;
1478 }
1479 
1480 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
1481 		const char *name)
1482 {
1483 	void __iomem *ret;
1484 	struct resource *res = platform_get_resource_byname(pdev,
1485 			IORESOURCE_MEM, name);
1486 
1487 	if (!res) {
1488 		DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name);
1489 		return ERR_PTR(-EINVAL);
1490 	}
1491 
1492 	ret = ioremap(res->start, resource_size(res));
1493 	if (!ret) {
1494 		DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name);
1495 		return ERR_PTR(-EINVAL);
1496 	}
1497 
1498 	return ret;
1499 }
1500 
1501 static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
1502 		const char *name, irq_handler_t handler)
1503 {
1504 	int irq, ret;
1505 
1506 	irq = platform_get_irq_byname(pdev, name);
1507 
1508 	ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH, name, gmu);
1509 	if (ret) {
1510 		DRM_DEV_ERROR(&pdev->dev, "Unable to get interrupt %s %d\n",
1511 			      name, ret);
1512 		return ret;
1513 	}
1514 
1515 	disable_irq(irq);
1516 
1517 	return irq;
1518 }
1519 
1520 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
1521 {
1522 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1523 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1524 	struct platform_device *pdev = to_platform_device(gmu->dev);
1525 
1526 	mutex_lock(&gmu->lock);
1527 	if (!gmu->initialized) {
1528 		mutex_unlock(&gmu->lock);
1529 		return;
1530 	}
1531 
1532 	gmu->initialized = false;
1533 
1534 	mutex_unlock(&gmu->lock);
1535 
1536 	pm_runtime_force_suspend(gmu->dev);
1537 
1538 	/*
1539 	 * Since cxpd is a virt device, the devlink with gmu-dev will be removed
1540 	 * automatically when we do detach
1541 	 */
1542 	dev_pm_domain_detach(gmu->cxpd, false);
1543 
1544 	if (!IS_ERR_OR_NULL(gmu->gxpd)) {
1545 		pm_runtime_disable(gmu->gxpd);
1546 		dev_pm_domain_detach(gmu->gxpd, false);
1547 	}
1548 
1549 	if (!IS_ERR_OR_NULL(gmu->qmp))
1550 		qmp_put(gmu->qmp);
1551 
1552 	iounmap(gmu->mmio);
1553 	if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
1554 		iounmap(gmu->rscc);
1555 	gmu->mmio = NULL;
1556 	gmu->rscc = NULL;
1557 
1558 	if (!adreno_has_gmu_wrapper(adreno_gpu)) {
1559 		a6xx_gmu_memory_free(gmu);
1560 
1561 		free_irq(gmu->gmu_irq, gmu);
1562 		free_irq(gmu->hfi_irq, gmu);
1563 	}
1564 
1565 	/* Drop reference taken in of_find_device_by_node */
1566 	put_device(gmu->dev);
1567 }
1568 
1569 static int cxpd_notifier_cb(struct notifier_block *nb,
1570 			unsigned long action, void *data)
1571 {
1572 	struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb);
1573 
1574 	if (action == GENPD_NOTIFY_OFF)
1575 		complete_all(&gmu->pd_gate);
1576 
1577 	return 0;
1578 }
1579 
1580 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
1581 {
1582 	struct platform_device *pdev = of_find_device_by_node(node);
1583 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1584 	int ret;
1585 
1586 	if (!pdev)
1587 		return -ENODEV;
1588 
1589 	gmu->dev = &pdev->dev;
1590 
1591 	of_dma_configure(gmu->dev, node, true);
1592 
1593 	pm_runtime_enable(gmu->dev);
1594 
1595 	/* Mark legacy for manual SPTPRAC control */
1596 	gmu->legacy = true;
1597 
1598 	/* Map the GMU registers */
1599 	gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
1600 	if (IS_ERR(gmu->mmio)) {
1601 		ret = PTR_ERR(gmu->mmio);
1602 		goto err_mmio;
1603 	}
1604 
1605 	gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
1606 	if (IS_ERR(gmu->cxpd)) {
1607 		ret = PTR_ERR(gmu->cxpd);
1608 		goto err_mmio;
1609 	}
1610 
1611 	if (!device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME)) {
1612 		ret = -ENODEV;
1613 		goto detach_cxpd;
1614 	}
1615 
1616 	init_completion(&gmu->pd_gate);
1617 	complete_all(&gmu->pd_gate);
1618 	gmu->pd_nb.notifier_call = cxpd_notifier_cb;
1619 
1620 	/* Get a link to the GX power domain to reset the GPU */
1621 	gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
1622 	if (IS_ERR(gmu->gxpd)) {
1623 		ret = PTR_ERR(gmu->gxpd);
1624 		goto err_mmio;
1625 	}
1626 
1627 	gmu->initialized = true;
1628 
1629 	return 0;
1630 
1631 detach_cxpd:
1632 	dev_pm_domain_detach(gmu->cxpd, false);
1633 
1634 err_mmio:
1635 	iounmap(gmu->mmio);
1636 
1637 	/* Drop reference taken in of_find_device_by_node */
1638 	put_device(gmu->dev);
1639 
1640 	return ret;
1641 }
1642 
1643 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
1644 {
1645 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1646 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1647 	struct platform_device *pdev = of_find_device_by_node(node);
1648 	struct device_link *link;
1649 	int ret;
1650 
1651 	if (!pdev)
1652 		return -ENODEV;
1653 
1654 	gmu->dev = &pdev->dev;
1655 
1656 	of_dma_configure(gmu->dev, node, true);
1657 
1658 	/* Fow now, don't do anything fancy until we get our feet under us */
1659 	gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
1660 
1661 	pm_runtime_enable(gmu->dev);
1662 
1663 	/* Get the list of clocks */
1664 	ret = a6xx_gmu_clocks_probe(gmu);
1665 	if (ret)
1666 		goto err_put_device;
1667 
1668 	ret = a6xx_gmu_memory_probe(gmu);
1669 	if (ret)
1670 		goto err_put_device;
1671 
1672 
1673 	/* A660 now requires handling "prealloc requests" in GMU firmware
1674 	 * For now just hardcode allocations based on the known firmware.
1675 	 * note: there is no indication that these correspond to "dummy" or
1676 	 * "debug" regions, but this "guess" allows reusing these BOs which
1677 	 * are otherwise unused by a660.
1678 	 */
1679 	gmu->dummy.size = SZ_4K;
1680 	if (adreno_is_a660_family(adreno_gpu) ||
1681 	    adreno_is_a7xx(adreno_gpu)) {
1682 		ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7,
1683 					    0x60400000, "debug");
1684 		if (ret)
1685 			goto err_memory;
1686 
1687 		gmu->dummy.size = SZ_8K;
1688 	}
1689 
1690 	/* Allocate memory for the GMU dummy page */
1691 	ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, gmu->dummy.size,
1692 				    0x60000000, "dummy");
1693 	if (ret)
1694 		goto err_memory;
1695 
1696 	/* Note that a650 family also includes a660 family: */
1697 	if (adreno_is_a650_family(adreno_gpu) ||
1698 	    adreno_is_a7xx(adreno_gpu)) {
1699 		ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
1700 			SZ_16M - SZ_16K, 0x04000, "icache");
1701 		if (ret)
1702 			goto err_memory;
1703 	/*
1704 	 * NOTE: when porting legacy ("pre-650-family") GPUs you may be tempted to add a condition
1705 	 * to allocate icache/dcache here, as per downstream code flow, but it may not actually be
1706 	 * necessary. If you omit this step and you don't get random pagefaults, you are likely
1707 	 * good to go without this!
1708 	 */
1709 	} else if (adreno_is_a640_family(adreno_gpu)) {
1710 		ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
1711 			SZ_256K - SZ_16K, 0x04000, "icache");
1712 		if (ret)
1713 			goto err_memory;
1714 
1715 		ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache,
1716 			SZ_256K - SZ_16K, 0x44000, "dcache");
1717 		if (ret)
1718 			goto err_memory;
1719 	} else if (adreno_is_a630_family(adreno_gpu)) {
1720 		/* HFI v1, has sptprac */
1721 		gmu->legacy = true;
1722 
1723 		/* Allocate memory for the GMU debug region */
1724 		ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0, "debug");
1725 		if (ret)
1726 			goto err_memory;
1727 	}
1728 
1729 	/* Allocate memory for the GMU log region */
1730 	ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_16K, 0, "log");
1731 	if (ret)
1732 		goto err_memory;
1733 
1734 	/* Allocate memory for for the HFI queues */
1735 	ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0, "hfi");
1736 	if (ret)
1737 		goto err_memory;
1738 
1739 	/* Map the GMU registers */
1740 	gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
1741 	if (IS_ERR(gmu->mmio)) {
1742 		ret = PTR_ERR(gmu->mmio);
1743 		goto err_memory;
1744 	}
1745 
1746 	if (adreno_is_a650_family(adreno_gpu) ||
1747 	    adreno_is_a7xx(adreno_gpu)) {
1748 		gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc");
1749 		if (IS_ERR(gmu->rscc)) {
1750 			ret = -ENODEV;
1751 			goto err_mmio;
1752 		}
1753 	} else {
1754 		gmu->rscc = gmu->mmio + 0x23000;
1755 	}
1756 
1757 	/* Get the HFI and GMU interrupts */
1758 	gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
1759 	gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
1760 
1761 	if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) {
1762 		ret = -ENODEV;
1763 		goto err_mmio;
1764 	}
1765 
1766 	gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
1767 	if (IS_ERR(gmu->cxpd)) {
1768 		ret = PTR_ERR(gmu->cxpd);
1769 		goto err_mmio;
1770 	}
1771 
1772 	link = device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME);
1773 	if (!link) {
1774 		ret = -ENODEV;
1775 		goto detach_cxpd;
1776 	}
1777 
1778 	gmu->qmp = qmp_get(gmu->dev);
1779 	if (IS_ERR(gmu->qmp) && adreno_is_a7xx(adreno_gpu)) {
1780 		ret = PTR_ERR(gmu->qmp);
1781 		goto remove_device_link;
1782 	}
1783 
1784 	init_completion(&gmu->pd_gate);
1785 	complete_all(&gmu->pd_gate);
1786 	gmu->pd_nb.notifier_call = cxpd_notifier_cb;
1787 
1788 	/*
1789 	 * Get a link to the GX power domain to reset the GPU in case of GMU
1790 	 * crash
1791 	 */
1792 	gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
1793 
1794 	/* Get the power levels for the GMU and GPU */
1795 	a6xx_gmu_pwrlevels_probe(gmu);
1796 
1797 	/* Set up the HFI queues */
1798 	a6xx_hfi_init(gmu);
1799 
1800 	/* Initialize RPMh */
1801 	a6xx_gmu_rpmh_init(gmu);
1802 
1803 	gmu->initialized = true;
1804 
1805 	return 0;
1806 
1807 remove_device_link:
1808 	device_link_del(link);
1809 
1810 detach_cxpd:
1811 	dev_pm_domain_detach(gmu->cxpd, false);
1812 
1813 err_mmio:
1814 	iounmap(gmu->mmio);
1815 	if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
1816 		iounmap(gmu->rscc);
1817 	free_irq(gmu->gmu_irq, gmu);
1818 	free_irq(gmu->hfi_irq, gmu);
1819 
1820 err_memory:
1821 	a6xx_gmu_memory_free(gmu);
1822 err_put_device:
1823 	/* Drop reference taken in of_find_device_by_node */
1824 	put_device(gmu->dev);
1825 
1826 	return ret;
1827 }
1828