1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
3 */
4
5 #include <linux/kernel.h>
6 #include <linux/types.h>
7 #include <linux/cpumask.h>
8 #include <linux/firmware/qcom/qcom_scm.h>
9 #include <linux/pm_opp.h>
10 #include <linux/nvmem-consumer.h>
11 #include <linux/slab.h>
12 #include "msm_gem.h"
13 #include "msm_mmu.h"
14 #include "a5xx_gpu.h"
15
16 extern bool hang_debug;
17 static void a5xx_dump(struct msm_gpu *gpu);
18
19 #define GPU_PAS_ID 13
20
update_shadow_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)21 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
22 {
23 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
24 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
25
26 if (a5xx_gpu->has_whereami) {
27 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
28 OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring)));
29 OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring)));
30 }
31 }
32
a5xx_flush(struct msm_gpu * gpu,struct msm_ringbuffer * ring,bool sync)33 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
34 bool sync)
35 {
36 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
37 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
38 uint32_t wptr;
39 unsigned long flags;
40
41 /*
42 * Most flush operations need to issue a WHERE_AM_I opcode to sync up
43 * the rptr shadow
44 */
45 if (sync)
46 update_shadow_rptr(gpu, ring);
47
48 spin_lock_irqsave(&ring->preempt_lock, flags);
49
50 /* Copy the shadow to the actual register */
51 ring->cur = ring->next;
52
53 /* Make sure to wrap wptr if we need to */
54 wptr = get_wptr(ring);
55
56 spin_unlock_irqrestore(&ring->preempt_lock, flags);
57
58 /* Make sure everything is posted before making a decision */
59 mb();
60
61 /* Update HW if this is the current ring and we are not in preempt */
62 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))
63 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
64 }
65
a5xx_submit_in_rb(struct msm_gpu * gpu,struct msm_gem_submit * submit)66 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)
67 {
68 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
69 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
70 struct msm_ringbuffer *ring = submit->ring;
71 struct drm_gem_object *obj;
72 uint32_t *ptr, dwords;
73 unsigned int i;
74
75 for (i = 0; i < submit->nr_cmds; i++) {
76 switch (submit->cmd[i].type) {
77 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
78 break;
79 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
80 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)
81 break;
82 fallthrough;
83 case MSM_SUBMIT_CMD_BUF:
84 /* copy commands into RB: */
85 obj = submit->bos[submit->cmd[i].idx].obj;
86 dwords = submit->cmd[i].size;
87
88 ptr = msm_gem_get_vaddr(obj);
89
90 /* _get_vaddr() shouldn't fail at this point,
91 * since we've already mapped it once in
92 * submit_reloc()
93 */
94 if (WARN_ON(IS_ERR_OR_NULL(ptr)))
95 return;
96
97 for (i = 0; i < dwords; i++) {
98 /* normally the OUT_PKTn() would wait
99 * for space for the packet. But since
100 * we just OUT_RING() the whole thing,
101 * need to call adreno_wait_ring()
102 * ourself:
103 */
104 adreno_wait_ring(ring, 1);
105 OUT_RING(ring, ptr[i]);
106 }
107
108 msm_gem_put_vaddr(obj);
109
110 break;
111 }
112 }
113
114 a5xx_gpu->last_seqno[ring->id] = submit->seqno;
115 a5xx_flush(gpu, ring, true);
116 a5xx_preempt_trigger(gpu);
117
118 /* we might not necessarily have a cmd from userspace to
119 * trigger an event to know that submit has completed, so
120 * do this manually:
121 */
122 a5xx_idle(gpu, ring);
123 ring->memptrs->fence = submit->seqno;
124 msm_gpu_retire(gpu);
125 }
126
a5xx_submit(struct msm_gpu * gpu,struct msm_gem_submit * submit)127 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
128 {
129 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
130 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
131 struct msm_ringbuffer *ring = submit->ring;
132 unsigned int i, ibs = 0;
133
134 adreno_check_and_reenable_stall(adreno_gpu);
135
136 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {
137 ring->cur_ctx_seqno = 0;
138 a5xx_submit_in_rb(gpu, submit);
139 return;
140 }
141
142 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
143 OUT_RING(ring, 0x02);
144
145 /* Turn off protected mode to write to special registers */
146 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
147 OUT_RING(ring, 0);
148
149 /* Set the save preemption record for the ring/command */
150 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
151 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
152 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
153
154 /* Turn back on protected mode */
155 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
156 OUT_RING(ring, 1);
157
158 /*
159 * Disable local preemption by default because it requires
160 * user-space to be aware of it and provide additional handling
161 * to restore rendering state or do various flushes on switch.
162 */
163 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
164 OUT_RING(ring, 0x0);
165
166 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */
167 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
168 OUT_RING(ring, 0x02);
169
170 /* Submit the commands */
171 for (i = 0; i < submit->nr_cmds; i++) {
172 switch (submit->cmd[i].type) {
173 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
174 break;
175 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
176 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)
177 break;
178 fallthrough;
179 case MSM_SUBMIT_CMD_BUF:
180 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
181 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
182 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
183 OUT_RING(ring, submit->cmd[i].size);
184 ibs++;
185 break;
186 }
187
188 /*
189 * Periodically update shadow-wptr if needed, so that we
190 * can see partial progress of submits with large # of
191 * cmds.. otherwise we could needlessly stall waiting for
192 * ringbuffer state, simply due to looking at a shadow
193 * rptr value that has not been updated
194 */
195 if ((ibs % 32) == 0)
196 update_shadow_rptr(gpu, ring);
197 }
198
199 /*
200 * Write the render mode to NULL (0) to indicate to the CP that the IBs
201 * are done rendering - otherwise a lucky preemption would start
202 * replaying from the last checkpoint
203 */
204 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5);
205 OUT_RING(ring, 0);
206 OUT_RING(ring, 0);
207 OUT_RING(ring, 0);
208 OUT_RING(ring, 0);
209 OUT_RING(ring, 0);
210
211 /* Turn off IB level preemptions */
212 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
213 OUT_RING(ring, 0x01);
214
215 /* Write the fence to the scratch register */
216 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
217 OUT_RING(ring, submit->seqno);
218 a5xx_gpu->last_seqno[ring->id] = submit->seqno;
219
220 /*
221 * Execute a CACHE_FLUSH_TS event. This will ensure that the
222 * timestamp is written to the memory and then triggers the interrupt
223 */
224 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
225 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
226 CP_EVENT_WRITE_0_IRQ);
227 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
228 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
229 OUT_RING(ring, submit->seqno);
230
231 /* Yield the floor on command completion */
232 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
233 /*
234 * If dword[2:1] are non zero, they specify an address for the CP to
235 * write the value of dword[3] to on preemption complete. Write 0 to
236 * skip the write
237 */
238 OUT_RING(ring, 0x00);
239 OUT_RING(ring, 0x00);
240 /* Data value - not used if the address above is 0 */
241 OUT_RING(ring, 0x01);
242 /* Set bit 0 to trigger an interrupt on preempt complete */
243 OUT_RING(ring, 0x01);
244
245 /* A WHERE_AM_I packet is not needed after a YIELD */
246 a5xx_flush(gpu, ring, false);
247
248 /* Check to see if we need to start preemption */
249 a5xx_preempt_trigger(gpu);
250 }
251
252 static const struct adreno_five_hwcg_regs {
253 u32 offset;
254 u32 value;
255 } a5xx_hwcg[] = {
256 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
257 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
258 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
259 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
260 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
261 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
262 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
263 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
264 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
265 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
266 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
267 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
268 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
269 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
270 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
271 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
272 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
273 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
274 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
275 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
276 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
277 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
278 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
279 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
280 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
281 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
282 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
283 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
284 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
285 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
286 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
287 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
288 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
289 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
290 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
291 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
292 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
293 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
294 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
295 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
296 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
297 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
298 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
299 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
300 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
301 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
302 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
303 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
304 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
305 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
306 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
307 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
308 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
309 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
310 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
311 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
312 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
313 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
314 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
315 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
316 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
317 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
318 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
319 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
320 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
321 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
322 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
323 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
324 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
325 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
326 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
327 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
328 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
329 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
330 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
331 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
332 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
333 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
334 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
335 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
336 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
337 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
338 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
339 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
340 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
341 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
342 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
343 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
344 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
345 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
346 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
347 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
348 }, a50x_hwcg[] = {
349 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
350 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
351 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
352 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
353 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
354 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
355 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
356 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
357 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
358 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
359 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
360 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
361 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
362 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
363 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
364 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
365 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
366 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4},
367 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
368 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
369 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
370 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
371 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
372 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
373 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
374 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
375 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
376 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
377 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
378 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
379 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
380 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
381 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
382 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
383 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
384 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
385 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
386 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
387 }, a512_hwcg[] = {
388 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
389 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
390 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
391 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
392 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
393 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
394 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
395 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
396 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
397 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
398 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
399 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
400 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
401 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
402 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
403 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
404 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
405 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
406 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
407 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
408 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
409 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
410 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
411 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
412 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
413 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
414 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
415 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
416 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
417 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
418 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
419 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
420 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
421 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
422 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
423 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
424 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
425 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
426 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
427 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
428 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
429 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
430 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
431 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
432 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
433 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
434 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
435 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
436 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
437 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
438 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
439 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
440 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
441 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
442 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
443 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
444 };
445
a5xx_set_hwcg(struct msm_gpu * gpu,bool state)446 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
447 {
448 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
449 const struct adreno_five_hwcg_regs *regs;
450 unsigned int i, sz;
451
452 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
453 adreno_is_a508(adreno_gpu)) {
454 regs = a50x_hwcg;
455 sz = ARRAY_SIZE(a50x_hwcg);
456 } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) {
457 regs = a512_hwcg;
458 sz = ARRAY_SIZE(a512_hwcg);
459 } else {
460 regs = a5xx_hwcg;
461 sz = ARRAY_SIZE(a5xx_hwcg);
462 }
463
464 for (i = 0; i < sz; i++)
465 gpu_write(gpu, regs[i].offset,
466 state ? regs[i].value : 0);
467
468 if (adreno_is_a540(adreno_gpu)) {
469 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);
470 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);
471 }
472
473 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
474 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
475 }
476
a5xx_me_init(struct msm_gpu * gpu)477 static int a5xx_me_init(struct msm_gpu *gpu)
478 {
479 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
480 struct msm_ringbuffer *ring = gpu->rb[0];
481
482 OUT_PKT7(ring, CP_ME_INIT, 8);
483
484 OUT_RING(ring, 0x0000002F);
485
486 /* Enable multiple hardware contexts */
487 OUT_RING(ring, 0x00000003);
488
489 /* Enable error detection */
490 OUT_RING(ring, 0x20000000);
491
492 /* Don't enable header dump */
493 OUT_RING(ring, 0x00000000);
494 OUT_RING(ring, 0x00000000);
495
496 /* Specify workarounds for various microcode issues */
497 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
498 adreno_is_a530(adreno_gpu)) {
499 /* Workaround for token end syncs
500 * Force a WFI after every direct-render 3D mode draw and every
501 * 2D mode 3 draw
502 */
503 OUT_RING(ring, 0x0000000B);
504 } else if (adreno_is_a510(adreno_gpu)) {
505 /* Workaround for token and syncs */
506 OUT_RING(ring, 0x00000001);
507 } else {
508 /* No workarounds enabled */
509 OUT_RING(ring, 0x00000000);
510 }
511
512 OUT_RING(ring, 0x00000000);
513 OUT_RING(ring, 0x00000000);
514
515 a5xx_flush(gpu, ring, true);
516 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
517 }
518
a5xx_preempt_start(struct msm_gpu * gpu)519 static int a5xx_preempt_start(struct msm_gpu *gpu)
520 {
521 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
522 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
523 struct msm_ringbuffer *ring = gpu->rb[0];
524
525 if (gpu->nr_rings == 1)
526 return 0;
527
528 /* Turn off protected mode to write to special registers */
529 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
530 OUT_RING(ring, 0);
531
532 /* Set the save preemption record for the ring/command */
533 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
534 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
535 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
536
537 /* Turn back on protected mode */
538 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
539 OUT_RING(ring, 1);
540
541 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
542 OUT_RING(ring, 0x00);
543
544 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
545 OUT_RING(ring, 0x01);
546
547 OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
548 OUT_RING(ring, 0x01);
549
550 /* Yield the floor on command completion */
551 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
552 OUT_RING(ring, 0x00);
553 OUT_RING(ring, 0x00);
554 OUT_RING(ring, 0x01);
555 OUT_RING(ring, 0x01);
556
557 /* The WHERE_AMI_I packet is not needed after a YIELD is issued */
558 a5xx_flush(gpu, ring, false);
559
560 return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
561 }
562
a5xx_ucode_check_version(struct a5xx_gpu * a5xx_gpu,struct drm_gem_object * obj)563 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu,
564 struct drm_gem_object *obj)
565 {
566 u32 *buf = msm_gem_get_vaddr(obj);
567
568 if (IS_ERR(buf))
569 return;
570
571 /*
572 * If the lowest nibble is 0xa that is an indication that this microcode
573 * has been patched. The actual version is in dword [3] but we only care
574 * about the patchlevel which is the lowest nibble of dword [3]
575 */
576 if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1)
577 a5xx_gpu->has_whereami = true;
578
579 msm_gem_put_vaddr(obj);
580 }
581
a5xx_ucode_load(struct msm_gpu * gpu)582 static int a5xx_ucode_load(struct msm_gpu *gpu)
583 {
584 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
585 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
586 int ret;
587
588 if (!a5xx_gpu->pm4_bo) {
589 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,
590 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);
591
592
593 if (IS_ERR(a5xx_gpu->pm4_bo)) {
594 ret = PTR_ERR(a5xx_gpu->pm4_bo);
595 a5xx_gpu->pm4_bo = NULL;
596 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",
597 ret);
598 return ret;
599 }
600
601 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw");
602 }
603
604 if (!a5xx_gpu->pfp_bo) {
605 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,
606 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);
607
608 if (IS_ERR(a5xx_gpu->pfp_bo)) {
609 ret = PTR_ERR(a5xx_gpu->pfp_bo);
610 a5xx_gpu->pfp_bo = NULL;
611 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",
612 ret);
613 return ret;
614 }
615
616 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw");
617 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);
618 }
619
620 if (a5xx_gpu->has_whereami) {
621 if (!a5xx_gpu->shadow_bo) {
622 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
623 sizeof(u32) * gpu->nr_rings,
624 MSM_BO_WC | MSM_BO_MAP_PRIV,
625 gpu->aspace, &a5xx_gpu->shadow_bo,
626 &a5xx_gpu->shadow_iova);
627
628 if (IS_ERR(a5xx_gpu->shadow))
629 return PTR_ERR(a5xx_gpu->shadow);
630
631 msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");
632 }
633 } else if (gpu->nr_rings > 1) {
634 /* Disable preemption if WHERE_AM_I isn't available */
635 a5xx_preempt_fini(gpu);
636 gpu->nr_rings = 1;
637 }
638
639 return 0;
640 }
641
642 #define SCM_GPU_ZAP_SHADER_RESUME 0
643
a5xx_zap_shader_resume(struct msm_gpu * gpu)644 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
645 {
646 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
647 int ret;
648
649 /*
650 * Adreno 506 have CPZ Retention feature and doesn't require
651 * to resume zap shader
652 */
653 if (adreno_is_a506(adreno_gpu))
654 return 0;
655
656 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
657 if (ret)
658 DRM_ERROR("%s: zap-shader resume failed: %d\n",
659 gpu->name, ret);
660
661 return ret;
662 }
663
a5xx_zap_shader_init(struct msm_gpu * gpu)664 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
665 {
666 static bool loaded;
667 int ret;
668
669 /*
670 * If the zap shader is already loaded into memory we just need to kick
671 * the remote processor to reinitialize it
672 */
673 if (loaded)
674 return a5xx_zap_shader_resume(gpu);
675
676 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
677
678 loaded = !ret;
679 return ret;
680 }
681
682 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
683 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
684 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
685 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
686 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
687 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
688 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
689 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
690 A5XX_RBBM_INT_0_MASK_CP_SW | \
691 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
692 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
693 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
694
a5xx_hw_init(struct msm_gpu * gpu)695 static int a5xx_hw_init(struct msm_gpu *gpu)
696 {
697 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
698 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
699 u32 hbb;
700 int ret;
701
702 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
703
704 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
705 adreno_is_a540(adreno_gpu))
706 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
707
708 /* Make all blocks contribute to the GPU BUSY perf counter */
709 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
710
711 /* Enable RBBM error reporting bits */
712 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
713
714 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
715 /*
716 * Mask out the activity signals from RB1-3 to avoid false
717 * positives
718 */
719
720 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
721 0xF0000000);
722 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
723 0xFFFFFFFF);
724 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
725 0xFFFFFFFF);
726 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
727 0xFFFFFFFF);
728 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
729 0xFFFFFFFF);
730 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
731 0xFFFFFFFF);
732 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
733 0xFFFFFFFF);
734 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
735 0xFFFFFFFF);
736 }
737
738 /* Enable fault detection */
739 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
740 (1 << 30) | 0xFFFF);
741
742 /* Turn on performance counters */
743 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
744
745 /* Select CP0 to always count cycles */
746 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
747
748 /* Select RBBM0 to countable 6 to get the busy status for devfreq */
749 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
750
751 /* Increase VFD cache access so LRZ and other data gets evicted less */
752 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
753
754 /* Disable L2 bypass in the UCHE */
755 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));
756 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));
757 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));
758 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));
759
760 /* Set the GMEM VA range (0 to gpu->gmem) */
761 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
762 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
763 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
764 0x00100000 + adreno_gpu->info->gmem - 1);
765 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
766
767 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
768 adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) {
769 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);
770 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
771 adreno_is_a508(adreno_gpu))
772 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
773 else
774 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);
775 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
776 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
777 } else {
778 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
779 if (adreno_is_a530(adreno_gpu))
780 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
781 else
782 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
783 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
784 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
785 }
786
787 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
788 adreno_is_a508(adreno_gpu))
789 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
790 (0x100 << 11 | 0x100 << 22));
791 else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) ||
792 adreno_is_a512(adreno_gpu))
793 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
794 (0x200 << 11 | 0x200 << 22));
795 else
796 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
797 (0x400 << 11 | 0x300 << 22));
798
799 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
800 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
801
802 /*
803 * Disable the RB sampler datapath DP2 clock gating optimization
804 * for 1-SP GPUs, as it is enabled by default.
805 */
806 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
807 adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) ||
808 adreno_is_a512(adreno_gpu))
809 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9));
810
811 /* Disable UCHE global filter as SP can invalidate/flush independently */
812 gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29));
813
814 /* Enable USE_RETENTION_FLOPS */
815 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
816
817 /* Enable ME/PFP split notification */
818 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
819
820 /*
821 * In A5x, CCU can send context_done event of a particular context to
822 * UCHE which ultimately reaches CP even when there is valid
823 * transaction of that context inside CCU. This can let CP to program
824 * config registers, which will make the "valid transaction" inside
825 * CCU to be interpreted differently. This can cause gpu fault. This
826 * bug is fixed in latest A510 revision. To enable this bug fix -
827 * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1
828 * (disable). For older A510 version this bit is unused.
829 */
830 if (adreno_is_a510(adreno_gpu))
831 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);
832
833 /* Enable HWCG */
834 a5xx_set_hwcg(gpu, true);
835
836 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
837
838 BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13);
839 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13;
840
841 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, hbb << 7);
842 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, hbb << 1);
843
844 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
845 adreno_is_a540(adreno_gpu))
846 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, hbb);
847
848 /* Disable All flat shading optimization (ALLFLATOPTDIS) */
849 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10));
850
851 /* Protect registers from the CP */
852 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
853
854 /* RBBM */
855 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
856 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
857 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
858 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
859 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
860 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
861
862 /* Content protect */
863 gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
864 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
865 16));
866 gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
867 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
868
869 /* CP */
870 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
871 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
872 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
873 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
874
875 /* RB */
876 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
877 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
878
879 /* VPC */
880 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
881 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16));
882
883 /* UCHE */
884 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
885
886 /* SMMU */
887 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
888 ADRENO_PROTECT_RW(0x10000, 0x8000));
889
890 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
891 /*
892 * Disable the trusted memory range - we don't actually supported secure
893 * memory rendering at this point in time and we don't want to block off
894 * part of the virtual memory space.
895 */
896 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000);
897 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
898
899 /* Put the GPU into 64 bit by default */
900 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);
901 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);
902 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
903 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);
904 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);
905 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
906 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);
907 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);
908 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
909 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);
910 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
911 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
912
913 /*
914 * VPC corner case with local memory load kill leads to corrupt
915 * internal state. Normal Disable does not work for all a5x chips.
916 * So do the following setting to disable it.
917 */
918 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) {
919 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));
920 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);
921 }
922
923 ret = adreno_hw_init(gpu);
924 if (ret)
925 return ret;
926
927 if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))
928 a5xx_gpmu_ucode_init(gpu);
929
930 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);
931 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);
932
933 /* Set the ringbuffer address */
934 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova);
935
936 /*
937 * If the microcode supports the WHERE_AM_I opcode then we can use that
938 * in lieu of the RPTR shadow and enable preemption. Otherwise, we
939 * can't safely use the RPTR shadow or preemption. In either case, the
940 * RPTR shadow should be disabled in hardware.
941 */
942 gpu_write(gpu, REG_A5XX_CP_RB_CNTL,
943 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
944
945 /* Configure the RPTR shadow if needed: */
946 if (a5xx_gpu->shadow_bo) {
947 gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,
948 shadowptr(a5xx_gpu, gpu->rb[0]));
949 }
950
951 a5xx_preempt_hw_init(gpu);
952
953 /* Disable the interrupts through the initial bringup stage */
954 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
955
956 /* Clear ME_HALT to start the micro engine */
957 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
958 ret = a5xx_me_init(gpu);
959 if (ret)
960 return ret;
961
962 ret = a5xx_power_init(gpu);
963 if (ret)
964 return ret;
965
966 /*
967 * Send a pipeline event stat to get misbehaving counters to start
968 * ticking correctly
969 */
970 if (adreno_is_a530(adreno_gpu)) {
971 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);
972 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));
973
974 a5xx_flush(gpu, gpu->rb[0], true);
975 if (!a5xx_idle(gpu, gpu->rb[0]))
976 return -EINVAL;
977 }
978
979 /*
980 * If the chip that we are using does support loading one, then
981 * try to load a zap shader into the secure world. If successful
982 * we can use the CP to switch out of secure mode. If not then we
983 * have no resource but to try to switch ourselves out manually. If we
984 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
985 * be blocked and a permissions violation will soon follow.
986 */
987 ret = a5xx_zap_shader_init(gpu);
988 if (!ret) {
989 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
990 OUT_RING(gpu->rb[0], 0x00000000);
991
992 a5xx_flush(gpu, gpu->rb[0], true);
993 if (!a5xx_idle(gpu, gpu->rb[0]))
994 return -EINVAL;
995 } else if (ret == -ENODEV) {
996 /*
997 * This device does not use zap shader (but print a warning
998 * just in case someone got their dt wrong.. hopefully they
999 * have a debug UART to realize the error of their ways...
1000 * if you mess this up you are about to crash horribly)
1001 */
1002 dev_warn_once(gpu->dev->dev,
1003 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
1004 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
1005 } else {
1006 return ret;
1007 }
1008
1009 /* Last step - yield the ringbuffer */
1010 a5xx_preempt_start(gpu);
1011
1012 return 0;
1013 }
1014
a5xx_recover(struct msm_gpu * gpu)1015 static void a5xx_recover(struct msm_gpu *gpu)
1016 {
1017 int i;
1018
1019 adreno_dump_info(gpu);
1020
1021 for (i = 0; i < 8; i++) {
1022 printk("CP_SCRATCH_REG%d: %u\n", i,
1023 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
1024 }
1025
1026 if (hang_debug)
1027 a5xx_dump(gpu);
1028
1029 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
1030 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
1031 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
1032 adreno_recover(gpu);
1033 }
1034
a5xx_destroy(struct msm_gpu * gpu)1035 static void a5xx_destroy(struct msm_gpu *gpu)
1036 {
1037 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1038 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1039
1040 DBG("%s", gpu->name);
1041
1042 a5xx_preempt_fini(gpu);
1043
1044 if (a5xx_gpu->pm4_bo) {
1045 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);
1046 drm_gem_object_put(a5xx_gpu->pm4_bo);
1047 }
1048
1049 if (a5xx_gpu->pfp_bo) {
1050 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);
1051 drm_gem_object_put(a5xx_gpu->pfp_bo);
1052 }
1053
1054 if (a5xx_gpu->gpmu_bo) {
1055 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
1056 drm_gem_object_put(a5xx_gpu->gpmu_bo);
1057 }
1058
1059 if (a5xx_gpu->shadow_bo) {
1060 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);
1061 drm_gem_object_put(a5xx_gpu->shadow_bo);
1062 }
1063
1064 adreno_gpu_cleanup(adreno_gpu);
1065 kfree(a5xx_gpu);
1066 }
1067
_a5xx_check_idle(struct msm_gpu * gpu)1068 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
1069 {
1070 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
1071 return false;
1072
1073 /*
1074 * Nearly every abnormality ends up pausing the GPU and triggering a
1075 * fault so we can safely just watch for this one interrupt to fire
1076 */
1077 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
1078 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
1079 }
1080
a5xx_idle(struct msm_gpu * gpu,struct msm_ringbuffer * ring)1081 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1082 {
1083 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1084 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1085
1086 if (ring != a5xx_gpu->cur_ring) {
1087 WARN(1, "Tried to idle a non-current ringbuffer\n");
1088 return false;
1089 }
1090
1091 /* wait for CP to drain ringbuffer: */
1092 if (!adreno_idle(gpu, ring))
1093 return false;
1094
1095 if (spin_until(_a5xx_check_idle(gpu))) {
1096 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
1097 gpu->name, __builtin_return_address(0),
1098 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1099 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),
1100 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1101 gpu_read(gpu, REG_A5XX_CP_RB_WPTR));
1102 return false;
1103 }
1104
1105 return true;
1106 }
1107
a5xx_fault_handler(void * arg,unsigned long iova,int flags,void * data)1108 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
1109 {
1110 struct msm_gpu *gpu = arg;
1111 struct adreno_smmu_fault_info *info = data;
1112 char block[12] = "unknown";
1113 u32 scratch[] = {
1114 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
1115 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
1116 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
1117 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)),
1118 };
1119
1120 if (info)
1121 snprintf(block, sizeof(block), "%x", info->fsynr1);
1122
1123 return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
1124 }
1125
a5xx_cp_err_irq(struct msm_gpu * gpu)1126 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
1127 {
1128 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
1129
1130 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
1131 u32 val;
1132
1133 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
1134
1135 /*
1136 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
1137 * read it twice
1138 */
1139
1140 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1141 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1142
1143 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
1144 val);
1145 }
1146
1147 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
1148 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
1149 gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
1150
1151 if (status & A5XX_CP_INT_CP_DMA_ERROR)
1152 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
1153
1154 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1155 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
1156
1157 dev_err_ratelimited(gpu->dev->dev,
1158 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1159 val & (1 << 24) ? "WRITE" : "READ",
1160 (val & 0xFFFFF) >> 2, val);
1161 }
1162
1163 if (status & A5XX_CP_INT_CP_AHB_ERROR) {
1164 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
1165 const char *access[16] = { "reserved", "reserved",
1166 "timestamp lo", "timestamp hi", "pfp read", "pfp write",
1167 "", "", "me read", "me write", "", "", "crashdump read",
1168 "crashdump write" };
1169
1170 dev_err_ratelimited(gpu->dev->dev,
1171 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
1172 status & 0xFFFFF, access[(status >> 24) & 0xF],
1173 (status & (1 << 31)), status);
1174 }
1175 }
1176
a5xx_rbbm_err_irq(struct msm_gpu * gpu,u32 status)1177 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
1178 {
1179 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
1180 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
1181
1182 dev_err_ratelimited(gpu->dev->dev,
1183 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
1184 val & (1 << 28) ? "WRITE" : "READ",
1185 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
1186 (val >> 24) & 0xF);
1187
1188 /* Clear the error */
1189 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
1190
1191 /* Clear the interrupt */
1192 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1193 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1194 }
1195
1196 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
1197 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
1198
1199 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
1200 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
1201 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
1202
1203 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
1204 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
1205 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
1206
1207 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
1208 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
1209 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
1210
1211 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1212 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
1213
1214 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1215 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
1216 }
1217
a5xx_uche_err_irq(struct msm_gpu * gpu)1218 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
1219 {
1220 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
1221
1222 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
1223
1224 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
1225 addr);
1226 }
1227
a5xx_gpmu_err_irq(struct msm_gpu * gpu)1228 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
1229 {
1230 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
1231 }
1232
a5xx_fault_detect_irq(struct msm_gpu * gpu)1233 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
1234 {
1235 struct drm_device *dev = gpu->dev;
1236 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1237
1238 /*
1239 * If stalled on SMMU fault, we could trip the GPU's hang detection,
1240 * but the fault handler will trigger the devcore dump, and we want
1241 * to otherwise resume normally rather than killing the submit, so
1242 * just bail.
1243 */
1244 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24))
1245 return;
1246
1247 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1248 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
1249 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1250 gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1251 gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
1252 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE),
1253 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
1254 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE),
1255 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
1256
1257 /* Turn off the hangcheck timer to keep it from bothering us */
1258 timer_delete(&gpu->hangcheck_timer);
1259
1260 kthread_queue_work(gpu->worker, &gpu->recover_work);
1261 }
1262
1263 #define RBBM_ERROR_MASK \
1264 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
1265 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
1266 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
1267 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
1268 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
1269 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1270
a5xx_irq(struct msm_gpu * gpu)1271 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
1272 {
1273 struct msm_drm_private *priv = gpu->dev->dev_private;
1274 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
1275
1276 /*
1277 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
1278 * before the source is cleared the interrupt will storm.
1279 */
1280 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1281 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1282
1283 if (priv->disable_err_irq) {
1284 status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS |
1285 A5XX_RBBM_INT_0_MASK_CP_SW;
1286 }
1287
1288 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
1289 if (status & RBBM_ERROR_MASK)
1290 a5xx_rbbm_err_irq(gpu, status);
1291
1292 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1293 a5xx_cp_err_irq(gpu);
1294
1295 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
1296 a5xx_fault_detect_irq(gpu);
1297
1298 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1299 a5xx_uche_err_irq(gpu);
1300
1301 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
1302 a5xx_gpmu_err_irq(gpu);
1303
1304 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1305 a5xx_preempt_trigger(gpu);
1306 msm_gpu_retire(gpu);
1307 }
1308
1309 if (status & A5XX_RBBM_INT_0_MASK_CP_SW)
1310 a5xx_preempt_irq(gpu);
1311
1312 return IRQ_HANDLED;
1313 }
1314
1315 static const u32 a5xx_registers[] = {
1316 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
1317 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
1318 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
1319 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
1320 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
1321 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
1322 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
1323 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
1324 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
1325 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
1326 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
1327 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
1328 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
1329 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
1330 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
1331 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
1332 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
1333 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
1334 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
1335 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
1336 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
1337 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
1338 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
1339 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
1340 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
1341 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
1342 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
1343 0xAC60, 0xAC60, ~0,
1344 };
1345
a5xx_dump(struct msm_gpu * gpu)1346 static void a5xx_dump(struct msm_gpu *gpu)
1347 {
1348 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n",
1349 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1350 adreno_dump(gpu);
1351 }
1352
a5xx_pm_resume(struct msm_gpu * gpu)1353 static int a5xx_pm_resume(struct msm_gpu *gpu)
1354 {
1355 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1356 int ret;
1357
1358 /* Turn on the core power */
1359 ret = msm_gpu_pm_resume(gpu);
1360 if (ret)
1361 return ret;
1362
1363 /* Adreno 505, 506, 508, 509, 510, 512 needs manual RBBM sus/res control */
1364 if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) {
1365 /* Halt the sp_input_clk at HM level */
1366 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);
1367 a5xx_set_hwcg(gpu, true);
1368 /* Turn on sp_input_clk at HM level */
1369 gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);
1370 return 0;
1371 }
1372
1373 /* Turn the RBCCU domain first to limit the chances of voltage droop */
1374 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
1375
1376 /* Wait 3 usecs before polling */
1377 udelay(3);
1378
1379 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
1380 (1 << 20), (1 << 20));
1381 if (ret) {
1382 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
1383 gpu->name,
1384 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
1385 return ret;
1386 }
1387
1388 /* Turn on the SP domain */
1389 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
1390 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
1391 (1 << 20), (1 << 20));
1392 if (ret)
1393 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
1394 gpu->name);
1395
1396 return ret;
1397 }
1398
a5xx_pm_suspend(struct msm_gpu * gpu)1399 static int a5xx_pm_suspend(struct msm_gpu *gpu)
1400 {
1401 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1402 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1403 u32 mask = 0xf;
1404 int i, ret;
1405
1406 /* A505, A506, A508, A510 have 3 XIN ports in VBIF */
1407 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
1408 adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu))
1409 mask = 0x7;
1410
1411 /* Clear the VBIF pipe before shutting down */
1412 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);
1413 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &
1414 mask) == mask);
1415
1416 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
1417
1418 /*
1419 * Reset the VBIF before power collapse to avoid issue with FIFO
1420 * entries on Adreno A510 and A530 (the others will tend to lock up)
1421 */
1422 if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) {
1423 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1424 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1425 }
1426
1427 ret = msm_gpu_pm_suspend(gpu);
1428 if (ret)
1429 return ret;
1430
1431 if (a5xx_gpu->has_whereami)
1432 for (i = 0; i < gpu->nr_rings; i++)
1433 a5xx_gpu->shadow[i] = 0;
1434
1435 return 0;
1436 }
1437
a5xx_get_timestamp(struct msm_gpu * gpu,uint64_t * value)1438 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1439 {
1440 *value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO);
1441
1442 return 0;
1443 }
1444
1445 struct a5xx_crashdumper {
1446 void *ptr;
1447 struct drm_gem_object *bo;
1448 u64 iova;
1449 };
1450
1451 struct a5xx_gpu_state {
1452 struct msm_gpu_state base;
1453 u32 *hlsqregs;
1454 };
1455
a5xx_crashdumper_init(struct msm_gpu * gpu,struct a5xx_crashdumper * dumper)1456 static int a5xx_crashdumper_init(struct msm_gpu *gpu,
1457 struct a5xx_crashdumper *dumper)
1458 {
1459 dumper->ptr = msm_gem_kernel_new(gpu->dev,
1460 SZ_1M, MSM_BO_WC, gpu->aspace,
1461 &dumper->bo, &dumper->iova);
1462
1463 if (!IS_ERR(dumper->ptr))
1464 msm_gem_object_set_name(dumper->bo, "crashdump");
1465
1466 return PTR_ERR_OR_ZERO(dumper->ptr);
1467 }
1468
a5xx_crashdumper_run(struct msm_gpu * gpu,struct a5xx_crashdumper * dumper)1469 static int a5xx_crashdumper_run(struct msm_gpu *gpu,
1470 struct a5xx_crashdumper *dumper)
1471 {
1472 u32 val;
1473
1474 if (IS_ERR_OR_NULL(dumper->ptr))
1475 return -EINVAL;
1476
1477 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova);
1478
1479 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
1480
1481 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
1482 val & 0x04, 100, 10000);
1483 }
1484
1485 /*
1486 * These are a list of the registers that need to be read through the HLSQ
1487 * aperture through the crashdumper. These are not nominally accessible from
1488 * the CPU on a secure platform.
1489 */
1490 static const struct {
1491 u32 type;
1492 u32 regoffset;
1493 u32 count;
1494 } a5xx_hlsq_aperture_regs[] = {
1495 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */
1496 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */
1497 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */
1498 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */
1499 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */
1500 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */
1501 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */
1502 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */
1503 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
1504 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
1505 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */
1506 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */
1507 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */
1508 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */
1509 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */
1510 };
1511
a5xx_gpu_state_get_hlsq_regs(struct msm_gpu * gpu,struct a5xx_gpu_state * a5xx_state)1512 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
1513 struct a5xx_gpu_state *a5xx_state)
1514 {
1515 struct a5xx_crashdumper dumper = { 0 };
1516 u32 offset, count = 0;
1517 u64 *ptr;
1518 int i;
1519
1520 if (a5xx_crashdumper_init(gpu, &dumper))
1521 return;
1522
1523 /* The script will be written at offset 0 */
1524 ptr = dumper.ptr;
1525
1526 /* Start writing the data at offset 256k */
1527 offset = dumper.iova + (256 * SZ_1K);
1528
1529 /* Count how many additional registers to get from the HLSQ aperture */
1530 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
1531 count += a5xx_hlsq_aperture_regs[i].count;
1532
1533 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
1534 if (!a5xx_state->hlsqregs)
1535 return;
1536
1537 /* Build the crashdump script */
1538 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1539 u32 type = a5xx_hlsq_aperture_regs[i].type;
1540 u32 c = a5xx_hlsq_aperture_regs[i].count;
1541
1542 /* Write the register to select the desired bank */
1543 *ptr++ = ((u64) type << 8);
1544 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
1545 (1 << 21) | 1;
1546
1547 *ptr++ = offset;
1548 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
1549 | c;
1550
1551 offset += c * sizeof(u32);
1552 }
1553
1554 /* Write two zeros to close off the script */
1555 *ptr++ = 0;
1556 *ptr++ = 0;
1557
1558 if (a5xx_crashdumper_run(gpu, &dumper)) {
1559 kfree(a5xx_state->hlsqregs);
1560 msm_gem_kernel_put(dumper.bo, gpu->aspace);
1561 return;
1562 }
1563
1564 /* Copy the data from the crashdumper to the state */
1565 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
1566 count * sizeof(u32));
1567
1568 msm_gem_kernel_put(dumper.bo, gpu->aspace);
1569 }
1570
a5xx_gpu_state_get(struct msm_gpu * gpu)1571 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
1572 {
1573 struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
1574 GFP_KERNEL);
1575 bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24));
1576
1577 if (!a5xx_state)
1578 return ERR_PTR(-ENOMEM);
1579
1580 /* Temporarily disable hardware clock gating before reading the hw */
1581 a5xx_set_hwcg(gpu, false);
1582
1583 /* First get the generic state from the adreno core */
1584 adreno_gpu_state_get(gpu, &(a5xx_state->base));
1585
1586 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
1587
1588 /*
1589 * Get the HLSQ regs with the help of the crashdumper, but only if
1590 * we are not stalled in an iommu fault (in which case the crashdumper
1591 * would not have access to memory)
1592 */
1593 if (!stalled)
1594 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
1595
1596 a5xx_set_hwcg(gpu, true);
1597
1598 return &a5xx_state->base;
1599 }
1600
a5xx_gpu_state_destroy(struct kref * kref)1601 static void a5xx_gpu_state_destroy(struct kref *kref)
1602 {
1603 struct msm_gpu_state *state = container_of(kref,
1604 struct msm_gpu_state, ref);
1605 struct a5xx_gpu_state *a5xx_state = container_of(state,
1606 struct a5xx_gpu_state, base);
1607
1608 kfree(a5xx_state->hlsqregs);
1609
1610 adreno_gpu_state_destroy(state);
1611 kfree(a5xx_state);
1612 }
1613
a5xx_gpu_state_put(struct msm_gpu_state * state)1614 static int a5xx_gpu_state_put(struct msm_gpu_state *state)
1615 {
1616 if (IS_ERR_OR_NULL(state))
1617 return 1;
1618
1619 return kref_put(&state->ref, a5xx_gpu_state_destroy);
1620 }
1621
1622
1623 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
a5xx_show(struct msm_gpu * gpu,struct msm_gpu_state * state,struct drm_printer * p)1624 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
1625 struct drm_printer *p)
1626 {
1627 int i, j;
1628 u32 pos = 0;
1629 struct a5xx_gpu_state *a5xx_state = container_of(state,
1630 struct a5xx_gpu_state, base);
1631
1632 if (IS_ERR_OR_NULL(state))
1633 return;
1634
1635 adreno_show(gpu, state, p);
1636
1637 /* Dump the additional a5xx HLSQ registers */
1638 if (!a5xx_state->hlsqregs)
1639 return;
1640
1641 drm_printf(p, "registers-hlsq:\n");
1642
1643 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1644 u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
1645 u32 c = a5xx_hlsq_aperture_regs[i].count;
1646
1647 for (j = 0; j < c; j++, pos++, o++) {
1648 /*
1649 * To keep the crashdump simple we pull the entire range
1650 * for each register type but not all of the registers
1651 * in the range are valid. Fortunately invalid registers
1652 * stick out like a sore thumb with a value of
1653 * 0xdeadbeef
1654 */
1655 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
1656 continue;
1657
1658 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n",
1659 o << 2, a5xx_state->hlsqregs[pos]);
1660 }
1661 }
1662 }
1663 #endif
1664
a5xx_active_ring(struct msm_gpu * gpu)1665 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
1666 {
1667 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1668 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1669
1670 return a5xx_gpu->cur_ring;
1671 }
1672
a5xx_gpu_busy(struct msm_gpu * gpu,unsigned long * out_sample_rate)1673 static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
1674 {
1675 u64 busy_cycles;
1676
1677 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO);
1678 *out_sample_rate = clk_get_rate(gpu->core_clk);
1679
1680 return busy_cycles;
1681 }
1682
a5xx_get_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)1683 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1684 {
1685 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1686 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1687
1688 if (a5xx_gpu->has_whereami)
1689 return a5xx_gpu->shadow[ring->id];
1690
1691 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);
1692 }
1693
1694 static const struct adreno_gpu_funcs funcs = {
1695 .base = {
1696 .get_param = adreno_get_param,
1697 .set_param = adreno_set_param,
1698 .hw_init = a5xx_hw_init,
1699 .ucode_load = a5xx_ucode_load,
1700 .pm_suspend = a5xx_pm_suspend,
1701 .pm_resume = a5xx_pm_resume,
1702 .recover = a5xx_recover,
1703 .submit = a5xx_submit,
1704 .active_ring = a5xx_active_ring,
1705 .irq = a5xx_irq,
1706 .destroy = a5xx_destroy,
1707 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1708 .show = a5xx_show,
1709 #endif
1710 #if defined(CONFIG_DEBUG_FS)
1711 .debugfs_init = a5xx_debugfs_init,
1712 #endif
1713 .gpu_busy = a5xx_gpu_busy,
1714 .gpu_state_get = a5xx_gpu_state_get,
1715 .gpu_state_put = a5xx_gpu_state_put,
1716 .create_address_space = adreno_create_address_space,
1717 .get_rptr = a5xx_get_rptr,
1718 },
1719 .get_timestamp = a5xx_get_timestamp,
1720 };
1721
check_speed_bin(struct device * dev)1722 static void check_speed_bin(struct device *dev)
1723 {
1724 struct nvmem_cell *cell;
1725 u32 val;
1726
1727 /*
1728 * If the OPP table specifies a opp-supported-hw property then we have
1729 * to set something with dev_pm_opp_set_supported_hw() or the table
1730 * doesn't get populated so pick an arbitrary value that should
1731 * ensure the default frequencies are selected but not conflict with any
1732 * actual bins
1733 */
1734 val = 0x80;
1735
1736 cell = nvmem_cell_get(dev, "speed_bin");
1737
1738 if (!IS_ERR(cell)) {
1739 void *buf = nvmem_cell_read(cell, NULL);
1740
1741 if (!IS_ERR(buf)) {
1742 u8 bin = *((u8 *) buf);
1743
1744 val = (1 << bin);
1745 kfree(buf);
1746 }
1747
1748 nvmem_cell_put(cell);
1749 }
1750
1751 devm_pm_opp_set_supported_hw(dev, &val, 1);
1752 }
1753
a5xx_gpu_init(struct drm_device * dev)1754 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1755 {
1756 struct msm_drm_private *priv = dev->dev_private;
1757 struct platform_device *pdev = priv->gpu_pdev;
1758 struct adreno_platform_config *config = pdev->dev.platform_data;
1759 struct a5xx_gpu *a5xx_gpu = NULL;
1760 struct adreno_gpu *adreno_gpu;
1761 struct msm_gpu *gpu;
1762 unsigned int nr_rings;
1763 int ret;
1764
1765 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1766 if (!a5xx_gpu)
1767 return ERR_PTR(-ENOMEM);
1768
1769 adreno_gpu = &a5xx_gpu->base;
1770 gpu = &adreno_gpu->base;
1771
1772 adreno_gpu->registers = a5xx_registers;
1773
1774 a5xx_gpu->lm_leakage = 0x4E001A;
1775
1776 check_speed_bin(&pdev->dev);
1777
1778 nr_rings = 4;
1779
1780 if (config->info->revn == 510)
1781 nr_rings = 1;
1782
1783 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, nr_rings);
1784 if (ret) {
1785 a5xx_destroy(&(a5xx_gpu->base.base));
1786 return ERR_PTR(ret);
1787 }
1788
1789 if (gpu->aspace)
1790 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
1791
1792 /* Set up the preemption specific bits and pieces for each ringbuffer */
1793 a5xx_preempt_init(gpu);
1794
1795 /* Set the highest bank bit */
1796 if (adreno_is_a540(adreno_gpu) || adreno_is_a530(adreno_gpu))
1797 adreno_gpu->ubwc_config.highest_bank_bit = 15;
1798 else
1799 adreno_gpu->ubwc_config.highest_bank_bit = 14;
1800
1801 /* a5xx only supports UBWC 1.0, these are not configurable */
1802 adreno_gpu->ubwc_config.macrotile_mode = 0;
1803 adreno_gpu->ubwc_config.ubwc_swizzle = 0x7;
1804
1805 adreno_gpu->uche_trap_base = 0x0001ffffffff0000ull;
1806
1807 return gpu;
1808 }
1809