1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 *
6 * Copyright (c) 2014 The Linux Foundation. All rights reserved.
7 */
8
9 #include "a3xx_gpu.h"
10
11 #define A3XX_INT0_MASK \
12 (A3XX_INT0_RBBM_AHB_ERROR | \
13 A3XX_INT0_RBBM_ATB_BUS_OVERFLOW | \
14 A3XX_INT0_CP_T0_PACKET_IN_IB | \
15 A3XX_INT0_CP_OPCODE_ERROR | \
16 A3XX_INT0_CP_RESERVED_BIT_ERROR | \
17 A3XX_INT0_CP_HW_FAULT | \
18 A3XX_INT0_CP_IB1_INT | \
19 A3XX_INT0_CP_IB2_INT | \
20 A3XX_INT0_CP_RB_INT | \
21 A3XX_INT0_CP_REG_PROTECT_FAULT | \
22 A3XX_INT0_CP_AHB_ERROR_HALT | \
23 A3XX_INT0_CACHE_FLUSH_TS | \
24 A3XX_INT0_UCHE_OOB_ACCESS)
25
26 extern bool hang_debug;
27
28 #define A3XX_VBIF_XIN_HALT_CTRL0_MASK GENMASK(5, 0)
29
30 static void a3xx_dump(struct msm_gpu *gpu);
31 static bool a3xx_idle(struct msm_gpu *gpu);
32
a3xx_submit(struct msm_gpu * gpu,struct msm_gem_submit * submit)33 static void a3xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
34 {
35 struct msm_ringbuffer *ring = submit->ring;
36 unsigned int i;
37
38 for (i = 0; i < submit->nr_cmds; i++) {
39 switch (submit->cmd[i].type) {
40 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
41 /* ignore IB-targets */
42 break;
43 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
44 /* ignore if there has not been a ctx switch: */
45 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)
46 break;
47 fallthrough;
48 case MSM_SUBMIT_CMD_BUF:
49 OUT_PKT3(ring, CP_INDIRECT_BUFFER_PFD, 2);
50 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
51 OUT_RING(ring, submit->cmd[i].size);
52 OUT_PKT2(ring);
53 break;
54 }
55 }
56
57 OUT_PKT0(ring, REG_AXXX_CP_SCRATCH_REG2, 1);
58 OUT_RING(ring, submit->seqno);
59
60 /* Flush HLSQ lazy updates to make sure there is nothing
61 * pending for indirect loads after the timestamp has
62 * passed:
63 */
64 OUT_PKT3(ring, CP_EVENT_WRITE, 1);
65 OUT_RING(ring, HLSQ_FLUSH);
66
67 /* wait for idle before cache flush/interrupt */
68 OUT_PKT3(ring, CP_WAIT_FOR_IDLE, 1);
69 OUT_RING(ring, 0x00000000);
70
71 /* BIT(31) of CACHE_FLUSH_TS triggers CACHE_FLUSH_TS IRQ from GPU */
72 OUT_PKT3(ring, CP_EVENT_WRITE, 3);
73 OUT_RING(ring, CACHE_FLUSH_TS | CP_EVENT_WRITE_0_IRQ);
74 OUT_RING(ring, rbmemptr(ring, fence));
75 OUT_RING(ring, submit->seqno);
76
77 #if 0
78 /* Dummy set-constant to trigger context rollover */
79 OUT_PKT3(ring, CP_SET_CONSTANT, 2);
80 OUT_RING(ring, CP_REG(REG_A3XX_HLSQ_CL_KERNEL_GROUP_X_REG));
81 OUT_RING(ring, 0x00000000);
82 #endif
83
84 adreno_flush(gpu, ring, REG_AXXX_CP_RB_WPTR);
85 }
86
a3xx_me_init(struct msm_gpu * gpu)87 static bool a3xx_me_init(struct msm_gpu *gpu)
88 {
89 struct msm_ringbuffer *ring = gpu->rb[0];
90
91 OUT_PKT3(ring, CP_ME_INIT, 17);
92 OUT_RING(ring, 0x000003f7);
93 OUT_RING(ring, 0x00000000);
94 OUT_RING(ring, 0x00000000);
95 OUT_RING(ring, 0x00000000);
96 OUT_RING(ring, 0x00000080);
97 OUT_RING(ring, 0x00000100);
98 OUT_RING(ring, 0x00000180);
99 OUT_RING(ring, 0x00006600);
100 OUT_RING(ring, 0x00000150);
101 OUT_RING(ring, 0x0000014e);
102 OUT_RING(ring, 0x00000154);
103 OUT_RING(ring, 0x00000001);
104 OUT_RING(ring, 0x00000000);
105 OUT_RING(ring, 0x00000000);
106 OUT_RING(ring, 0x00000000);
107 OUT_RING(ring, 0x00000000);
108 OUT_RING(ring, 0x00000000);
109
110 adreno_flush(gpu, ring, REG_AXXX_CP_RB_WPTR);
111 return a3xx_idle(gpu);
112 }
113
a3xx_hw_init(struct msm_gpu * gpu)114 static int a3xx_hw_init(struct msm_gpu *gpu)
115 {
116 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
117 struct a3xx_gpu *a3xx_gpu = to_a3xx_gpu(adreno_gpu);
118 uint32_t *ptr, len;
119 int i, ret;
120
121 DBG("%s", gpu->name);
122
123 if (adreno_is_a305(adreno_gpu)) {
124 /* Set up 16 deep read/write request queues: */
125 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF0, 0x10101010);
126 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF1, 0x10101010);
127 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x10101010);
128 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x10101010);
129 gpu_write(gpu, REG_A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303);
130 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF0, 0x10101010);
131 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF1, 0x10101010);
132 /* Enable WR-REQ: */
133 gpu_write(gpu, REG_A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x0000ff);
134 /* Set up round robin arbitration between both AXI ports: */
135 gpu_write(gpu, REG_A3XX_VBIF_ARB_CTL, 0x00000030);
136 /* Set up AOOO: */
137 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003c);
138 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO, 0x003c003c);
139 } else if (adreno_is_a305b(adreno_gpu)) {
140 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF0, 0x00181818);
141 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF0, 0x00181818);
142 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000018);
143 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000018);
144 gpu_write(gpu, REG_A3XX_VBIF_DDR_OUT_MAX_BURST, 0x00000303);
145 gpu_write(gpu, REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003);
146 } else if (adreno_is_a306(adreno_gpu)) {
147 gpu_write(gpu, REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003);
148 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x0000000a);
149 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x0000000a);
150 } else if (adreno_is_a306a(adreno_gpu)) {
151 gpu_write(gpu, REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003);
152 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000010);
153 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000010);
154 } else if (adreno_is_a320(adreno_gpu)) {
155 /* Set up 16 deep read/write request queues: */
156 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF0, 0x10101010);
157 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF1, 0x10101010);
158 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x10101010);
159 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x10101010);
160 gpu_write(gpu, REG_A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303);
161 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF0, 0x10101010);
162 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF1, 0x10101010);
163 /* Enable WR-REQ: */
164 gpu_write(gpu, REG_A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x0000ff);
165 /* Set up round robin arbitration between both AXI ports: */
166 gpu_write(gpu, REG_A3XX_VBIF_ARB_CTL, 0x00000030);
167 /* Set up AOOO: */
168 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003c);
169 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO, 0x003c003c);
170 /* Enable 1K sort: */
171 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT, 0x000000ff);
172 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT_CONF, 0x000000a4);
173
174 } else if (adreno_is_a330v2(adreno_gpu)) {
175 /*
176 * Most of the VBIF registers on 8974v2 have the correct
177 * values at power on, so we won't modify those if we don't
178 * need to
179 */
180 /* Enable 1k sort: */
181 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT, 0x0001003f);
182 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT_CONF, 0x000000a4);
183 /* Enable WR-REQ: */
184 gpu_write(gpu, REG_A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x00003f);
185 gpu_write(gpu, REG_A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303);
186 /* Set up VBIF_ROUND_ROBIN_QOS_ARB: */
187 gpu_write(gpu, REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003);
188
189 } else if (adreno_is_a330(adreno_gpu)) {
190 /* Set up 16 deep read/write request queues: */
191 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF0, 0x18181818);
192 gpu_write(gpu, REG_A3XX_VBIF_IN_RD_LIM_CONF1, 0x18181818);
193 gpu_write(gpu, REG_A3XX_VBIF_OUT_RD_LIM_CONF0, 0x18181818);
194 gpu_write(gpu, REG_A3XX_VBIF_OUT_WR_LIM_CONF0, 0x18181818);
195 gpu_write(gpu, REG_A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303);
196 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF0, 0x18181818);
197 gpu_write(gpu, REG_A3XX_VBIF_IN_WR_LIM_CONF1, 0x18181818);
198 /* Enable WR-REQ: */
199 gpu_write(gpu, REG_A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x00003f);
200 /* Set up round robin arbitration between both AXI ports: */
201 gpu_write(gpu, REG_A3XX_VBIF_ARB_CTL, 0x00000030);
202 /* Set up VBIF_ROUND_ROBIN_QOS_ARB: */
203 gpu_write(gpu, REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0001);
204 /* Set up AOOO: */
205 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003f);
206 gpu_write(gpu, REG_A3XX_VBIF_OUT_AXI_AOOO, 0x003f003f);
207 /* Enable 1K sort: */
208 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT, 0x0001003f);
209 gpu_write(gpu, REG_A3XX_VBIF_ABIT_SORT_CONF, 0x000000a4);
210 /* Disable VBIF clock gating. This is to enable AXI running
211 * higher frequency than GPU:
212 */
213 gpu_write(gpu, REG_A3XX_VBIF_CLKON, 0x00000001);
214
215 } else {
216 BUG();
217 }
218
219 /* Make all blocks contribute to the GPU BUSY perf counter: */
220 gpu_write(gpu, REG_A3XX_RBBM_GPU_BUSY_MASKED, 0xffffffff);
221
222 /* Tune the hystersis counters for SP and CP idle detection: */
223 gpu_write(gpu, REG_A3XX_RBBM_SP_HYST_CNT, 0x10);
224 gpu_write(gpu, REG_A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL, 0x10);
225
226 /* Enable the RBBM error reporting bits. This lets us get
227 * useful information on failure:
228 */
229 gpu_write(gpu, REG_A3XX_RBBM_AHB_CTL0, 0x00000001);
230
231 /* Enable AHB error reporting: */
232 gpu_write(gpu, REG_A3XX_RBBM_AHB_CTL1, 0xa6ffffff);
233
234 /* Turn on the power counters: */
235 gpu_write(gpu, REG_A3XX_RBBM_RBBM_CTL, 0x00030000);
236
237 /* Turn on hang detection - this spews a lot of useful information
238 * into the RBBM registers on a hang:
239 */
240 gpu_write(gpu, REG_A3XX_RBBM_INTERFACE_HANG_INT_CTL, 0x00010fff);
241
242 /* Enable 64-byte cacheline size. HW Default is 32-byte (0x000000E0): */
243 gpu_write(gpu, REG_A3XX_UCHE_CACHE_MODE_CONTROL_REG, 0x00000001);
244
245 /* Enable Clock gating: */
246 if (adreno_is_a305b(adreno_gpu) ||
247 adreno_is_a306(adreno_gpu) ||
248 adreno_is_a306a(adreno_gpu))
249 gpu_write(gpu, REG_A3XX_RBBM_CLOCK_CTL, 0xaaaaaaaa);
250 else if (adreno_is_a320(adreno_gpu))
251 gpu_write(gpu, REG_A3XX_RBBM_CLOCK_CTL, 0xbfffffff);
252 else if (adreno_is_a330v2(adreno_gpu))
253 gpu_write(gpu, REG_A3XX_RBBM_CLOCK_CTL, 0xaaaaaaaa);
254 else if (adreno_is_a330(adreno_gpu))
255 gpu_write(gpu, REG_A3XX_RBBM_CLOCK_CTL, 0xbffcffff);
256
257 if (adreno_is_a330v2(adreno_gpu))
258 gpu_write(gpu, REG_A3XX_RBBM_GPR0_CTL, 0x05515455);
259 else if (adreno_is_a330(adreno_gpu))
260 gpu_write(gpu, REG_A3XX_RBBM_GPR0_CTL, 0x00000000);
261
262 /* Set the OCMEM base address for A330, etc */
263 if (a3xx_gpu->ocmem.hdl) {
264 gpu_write(gpu, REG_A3XX_RB_GMEM_BASE_ADDR,
265 (unsigned int)(a3xx_gpu->ocmem.base >> 14));
266 }
267
268 /* Turn on performance counters: */
269 gpu_write(gpu, REG_A3XX_RBBM_PERFCTR_CTL, 0x01);
270
271 gpu_write(gpu, REG_A3XX_RBBM_INT_0_MASK, A3XX_INT0_MASK);
272
273 ret = adreno_hw_init(gpu);
274 if (ret)
275 return ret;
276
277 /*
278 * Use the default ringbuffer size and block size but disable the RPTR
279 * shadow
280 */
281 gpu_write(gpu, REG_AXXX_CP_RB_CNTL,
282 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
283
284 /* Set the ringbuffer address */
285 gpu_write(gpu, REG_AXXX_CP_RB_BASE, lower_32_bits(gpu->rb[0]->iova));
286
287 /* setup access protection: */
288 gpu_write(gpu, REG_A3XX_CP_PROTECT_CTRL, 0x00000007);
289
290 /* RBBM registers */
291 gpu_write(gpu, REG_A3XX_CP_PROTECT(0), 0x63000040);
292 gpu_write(gpu, REG_A3XX_CP_PROTECT(1), 0x62000080);
293 gpu_write(gpu, REG_A3XX_CP_PROTECT(2), 0x600000cc);
294 gpu_write(gpu, REG_A3XX_CP_PROTECT(3), 0x60000108);
295 gpu_write(gpu, REG_A3XX_CP_PROTECT(4), 0x64000140);
296 gpu_write(gpu, REG_A3XX_CP_PROTECT(5), 0x66000400);
297
298 /* CP registers */
299 gpu_write(gpu, REG_A3XX_CP_PROTECT(6), 0x65000700);
300 gpu_write(gpu, REG_A3XX_CP_PROTECT(7), 0x610007d8);
301 gpu_write(gpu, REG_A3XX_CP_PROTECT(8), 0x620007e0);
302 gpu_write(gpu, REG_A3XX_CP_PROTECT(9), 0x61001178);
303 gpu_write(gpu, REG_A3XX_CP_PROTECT(10), 0x64001180);
304
305 /* RB registers */
306 gpu_write(gpu, REG_A3XX_CP_PROTECT(11), 0x60003300);
307
308 /* VBIF registers */
309 gpu_write(gpu, REG_A3XX_CP_PROTECT(12), 0x6b00c000);
310
311 /* NOTE: PM4/micro-engine firmware registers look to be the same
312 * for a2xx and a3xx.. we could possibly push that part down to
313 * adreno_gpu base class. Or push both PM4 and PFP but
314 * parameterize the pfp ucode addr/data registers..
315 */
316
317 /* Load PM4: */
318 ptr = (uint32_t *)(adreno_gpu->fw[ADRENO_FW_PM4]->data);
319 len = adreno_gpu->fw[ADRENO_FW_PM4]->size / 4;
320 DBG("loading PM4 ucode version: %x", ptr[1]);
321
322 gpu_write(gpu, REG_AXXX_CP_DEBUG,
323 AXXX_CP_DEBUG_DYNAMIC_CLK_DISABLE |
324 AXXX_CP_DEBUG_MIU_128BIT_WRITE_ENABLE);
325 gpu_write(gpu, REG_AXXX_CP_ME_RAM_WADDR, 0);
326 for (i = 1; i < len; i++)
327 gpu_write(gpu, REG_AXXX_CP_ME_RAM_DATA, ptr[i]);
328
329 /* Load PFP: */
330 ptr = (uint32_t *)(adreno_gpu->fw[ADRENO_FW_PFP]->data);
331 len = adreno_gpu->fw[ADRENO_FW_PFP]->size / 4;
332 DBG("loading PFP ucode version: %x", ptr[5]);
333
334 gpu_write(gpu, REG_A3XX_CP_PFP_UCODE_ADDR, 0);
335 for (i = 1; i < len; i++)
336 gpu_write(gpu, REG_A3XX_CP_PFP_UCODE_DATA, ptr[i]);
337
338 /* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */
339 if (adreno_is_a305(adreno_gpu) ||
340 adreno_is_a306(adreno_gpu) ||
341 adreno_is_a306a(adreno_gpu) ||
342 adreno_is_a320(adreno_gpu)) {
343 gpu_write(gpu, REG_AXXX_CP_QUEUE_THRESHOLDS,
344 AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START(2) |
345 AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START(6) |
346 AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START(14));
347 } else if (adreno_is_a330(adreno_gpu) || adreno_is_a305b(adreno_gpu)) {
348 /* NOTE: this (value take from downstream android driver)
349 * includes some bits outside of the known bitfields. But
350 * A330 has this "MERCIU queue" thing too, which might
351 * explain a new bitfield or reshuffling:
352 */
353 gpu_write(gpu, REG_AXXX_CP_QUEUE_THRESHOLDS, 0x003e2008);
354 }
355
356 /* clear ME_HALT to start micro engine */
357 gpu_write(gpu, REG_AXXX_CP_ME_CNTL, 0);
358
359 return a3xx_me_init(gpu) ? 0 : -EINVAL;
360 }
361
a3xx_recover(struct msm_gpu * gpu)362 static void a3xx_recover(struct msm_gpu *gpu)
363 {
364 int i;
365
366 adreno_dump_info(gpu);
367
368 for (i = 0; i < 8; i++) {
369 printk("CP_SCRATCH_REG%d: %u\n", i,
370 gpu_read(gpu, REG_AXXX_CP_SCRATCH_REG0 + i));
371 }
372
373 /* dump registers before resetting gpu, if enabled: */
374 if (hang_debug)
375 a3xx_dump(gpu);
376
377 gpu_write(gpu, REG_A3XX_RBBM_SW_RESET_CMD, 1);
378 gpu_read(gpu, REG_A3XX_RBBM_SW_RESET_CMD);
379 gpu_write(gpu, REG_A3XX_RBBM_SW_RESET_CMD, 0);
380 adreno_recover(gpu);
381 }
382
a3xx_destroy(struct msm_gpu * gpu)383 static void a3xx_destroy(struct msm_gpu *gpu)
384 {
385 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
386 struct a3xx_gpu *a3xx_gpu = to_a3xx_gpu(adreno_gpu);
387
388 DBG("%s", gpu->name);
389
390 adreno_gpu_cleanup(adreno_gpu);
391
392 adreno_gpu_ocmem_cleanup(&a3xx_gpu->ocmem);
393
394 kfree(a3xx_gpu);
395 }
396
a3xx_idle(struct msm_gpu * gpu)397 static bool a3xx_idle(struct msm_gpu *gpu)
398 {
399 /* wait for ringbuffer to drain: */
400 if (!adreno_idle(gpu, gpu->rb[0]))
401 return false;
402
403 /* then wait for GPU to finish: */
404 if (spin_until(!(gpu_read(gpu, REG_A3XX_RBBM_STATUS) &
405 A3XX_RBBM_STATUS_GPU_BUSY))) {
406 DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name);
407
408 /* TODO maybe we need to reset GPU here to recover from hang? */
409 return false;
410 }
411
412 return true;
413 }
414
a3xx_irq(struct msm_gpu * gpu)415 static irqreturn_t a3xx_irq(struct msm_gpu *gpu)
416 {
417 uint32_t status;
418
419 status = gpu_read(gpu, REG_A3XX_RBBM_INT_0_STATUS);
420 DBG("%s: %08x", gpu->name, status);
421
422 // TODO
423
424 gpu_write(gpu, REG_A3XX_RBBM_INT_CLEAR_CMD, status);
425
426 msm_gpu_retire(gpu);
427
428 return IRQ_HANDLED;
429 }
430
431 static const unsigned int a3xx_registers[] = {
432 0x0000, 0x0002, 0x0010, 0x0012, 0x0018, 0x0018, 0x0020, 0x0027,
433 0x0029, 0x002b, 0x002e, 0x0033, 0x0040, 0x0042, 0x0050, 0x005c,
434 0x0060, 0x006c, 0x0080, 0x0082, 0x0084, 0x0088, 0x0090, 0x00e5,
435 0x00ea, 0x00ed, 0x0100, 0x0100, 0x0110, 0x0123, 0x01c0, 0x01c1,
436 0x01c3, 0x01c5, 0x01c7, 0x01c7, 0x01d5, 0x01d9, 0x01dc, 0x01dd,
437 0x01ea, 0x01ea, 0x01ee, 0x01f1, 0x01f5, 0x01f5, 0x01fc, 0x01ff,
438 0x0440, 0x0440, 0x0443, 0x0443, 0x0445, 0x0445, 0x044d, 0x044f,
439 0x0452, 0x0452, 0x0454, 0x046f, 0x047c, 0x047c, 0x047f, 0x047f,
440 0x0578, 0x057f, 0x0600, 0x0602, 0x0605, 0x0607, 0x060a, 0x060e,
441 0x0612, 0x0614, 0x0c01, 0x0c02, 0x0c06, 0x0c1d, 0x0c3d, 0x0c3f,
442 0x0c48, 0x0c4b, 0x0c80, 0x0c80, 0x0c88, 0x0c8b, 0x0ca0, 0x0cb7,
443 0x0cc0, 0x0cc1, 0x0cc6, 0x0cc7, 0x0ce4, 0x0ce5, 0x0e00, 0x0e05,
444 0x0e0c, 0x0e0c, 0x0e22, 0x0e23, 0x0e41, 0x0e45, 0x0e64, 0x0e65,
445 0x0e80, 0x0e82, 0x0e84, 0x0e89, 0x0ea0, 0x0ea1, 0x0ea4, 0x0ea7,
446 0x0ec4, 0x0ecb, 0x0ee0, 0x0ee0, 0x0f00, 0x0f01, 0x0f03, 0x0f09,
447 0x2040, 0x2040, 0x2044, 0x2044, 0x2048, 0x204d, 0x2068, 0x2069,
448 0x206c, 0x206d, 0x2070, 0x2070, 0x2072, 0x2072, 0x2074, 0x2075,
449 0x2079, 0x207a, 0x20c0, 0x20d3, 0x20e4, 0x20ef, 0x2100, 0x2109,
450 0x210c, 0x210c, 0x210e, 0x210e, 0x2110, 0x2111, 0x2114, 0x2115,
451 0x21e4, 0x21e4, 0x21ea, 0x21ea, 0x21ec, 0x21ed, 0x21f0, 0x21f0,
452 0x2200, 0x2212, 0x2214, 0x2217, 0x221a, 0x221a, 0x2240, 0x227e,
453 0x2280, 0x228b, 0x22c0, 0x22c0, 0x22c4, 0x22ce, 0x22d0, 0x22d8,
454 0x22df, 0x22e6, 0x22e8, 0x22e9, 0x22ec, 0x22ec, 0x22f0, 0x22f7,
455 0x22ff, 0x22ff, 0x2340, 0x2343, 0x2440, 0x2440, 0x2444, 0x2444,
456 0x2448, 0x244d, 0x2468, 0x2469, 0x246c, 0x246d, 0x2470, 0x2470,
457 0x2472, 0x2472, 0x2474, 0x2475, 0x2479, 0x247a, 0x24c0, 0x24d3,
458 0x24e4, 0x24ef, 0x2500, 0x2509, 0x250c, 0x250c, 0x250e, 0x250e,
459 0x2510, 0x2511, 0x2514, 0x2515, 0x25e4, 0x25e4, 0x25ea, 0x25ea,
460 0x25ec, 0x25ed, 0x25f0, 0x25f0, 0x2600, 0x2612, 0x2614, 0x2617,
461 0x261a, 0x261a, 0x2640, 0x267e, 0x2680, 0x268b, 0x26c0, 0x26c0,
462 0x26c4, 0x26ce, 0x26d0, 0x26d8, 0x26df, 0x26e6, 0x26e8, 0x26e9,
463 0x26ec, 0x26ec, 0x26f0, 0x26f7, 0x26ff, 0x26ff, 0x2740, 0x2743,
464 0x300c, 0x300e, 0x301c, 0x301d, 0x302a, 0x302a, 0x302c, 0x302d,
465 0x3030, 0x3031, 0x3034, 0x3036, 0x303c, 0x303c, 0x305e, 0x305f,
466 ~0 /* sentinel */
467 };
468
469 /* would be nice to not have to duplicate the _show() stuff with printk(): */
a3xx_dump(struct msm_gpu * gpu)470 static void a3xx_dump(struct msm_gpu *gpu)
471 {
472 printk("status: %08x\n",
473 gpu_read(gpu, REG_A3XX_RBBM_STATUS));
474 adreno_dump(gpu);
475 }
476
a3xx_gpu_state_get(struct msm_gpu * gpu)477 static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
478 {
479 struct msm_gpu_state *state = kzalloc_obj(*state);
480
481 if (!state)
482 return ERR_PTR(-ENOMEM);
483
484 adreno_gpu_state_get(gpu, state);
485
486 state->rbbm_status = gpu_read(gpu, REG_A3XX_RBBM_STATUS);
487
488 return state;
489 }
490
a3xx_gpu_busy(struct msm_gpu * gpu,unsigned long * out_sample_rate)491 static u64 a3xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
492 {
493 u64 busy_cycles;
494
495 busy_cycles = gpu_read64(gpu, REG_A3XX_RBBM_PERFCTR_RBBM_1_LO);
496 *out_sample_rate = clk_get_rate(gpu->core_clk);
497
498 return busy_cycles;
499 }
500
a3xx_vbif_halt(struct msm_gpu * gpu)501 static int a3xx_vbif_halt(struct msm_gpu *gpu)
502 {
503 u32 ack;
504 int ret;
505
506 gpu_write(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL0,
507 A3XX_VBIF_XIN_HALT_CTRL0_MASK);
508 ret = spin_until(((ack = gpu_read(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL1)) &
509 A3XX_VBIF_XIN_HALT_CTRL0_MASK) ==
510 A3XX_VBIF_XIN_HALT_CTRL0_MASK);
511 gpu_write(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL0, 0);
512
513 if (ret)
514 return -EBUSY;
515
516 return 0;
517 }
518
a3xx_pm_suspend(struct msm_gpu * gpu)519 static int a3xx_pm_suspend(struct msm_gpu *gpu)
520 {
521 int ret;
522
523 if (!a3xx_idle(gpu))
524 return -EBUSY;
525
526 ret = a3xx_vbif_halt(gpu);
527 if (ret)
528 return ret;
529
530 return msm_gpu_pm_suspend(gpu);
531 }
532
a3xx_get_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)533 static u32 a3xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
534 {
535 ring->memptrs->rptr = gpu_read(gpu, REG_AXXX_CP_RB_RPTR);
536 return ring->memptrs->rptr;
537 }
538
a3xx_gpu_init(struct drm_device * dev)539 static struct msm_gpu *a3xx_gpu_init(struct drm_device *dev)
540 {
541 struct a3xx_gpu *a3xx_gpu = NULL;
542 struct adreno_gpu *adreno_gpu;
543 struct msm_gpu *gpu;
544 struct msm_drm_private *priv = dev->dev_private;
545 struct platform_device *pdev = priv->gpu_pdev;
546 struct adreno_platform_config *config = pdev->dev.platform_data;
547 struct icc_path *ocmem_icc_path;
548 struct icc_path *icc_path;
549 int ret;
550
551 if (!pdev) {
552 DRM_DEV_ERROR(dev->dev, "no a3xx device\n");
553 ret = -ENXIO;
554 goto fail;
555 }
556
557 a3xx_gpu = kzalloc_obj(*a3xx_gpu);
558 if (!a3xx_gpu) {
559 ret = -ENOMEM;
560 goto fail;
561 }
562
563 adreno_gpu = &a3xx_gpu->base;
564 gpu = &adreno_gpu->base;
565
566 adreno_gpu->registers = a3xx_registers;
567
568 ret = adreno_gpu_init(dev, pdev, adreno_gpu, config->info->funcs, 1);
569 if (ret)
570 goto fail;
571
572 /* if needed, allocate gmem: */
573 if (adreno_is_a330(adreno_gpu) || adreno_is_a305b(adreno_gpu)) {
574 ret = adreno_gpu_ocmem_init(&adreno_gpu->base.pdev->dev,
575 adreno_gpu, &a3xx_gpu->ocmem);
576 if (ret)
577 goto fail;
578 }
579
580 icc_path = devm_of_icc_get(&pdev->dev, "gfx-mem");
581 if (IS_ERR(icc_path)) {
582 ret = PTR_ERR(icc_path);
583 goto fail;
584 }
585
586 ocmem_icc_path = devm_of_icc_get(&pdev->dev, "ocmem");
587 if (IS_ERR(ocmem_icc_path)) {
588 ret = PTR_ERR(ocmem_icc_path);
589 /* allow -ENODATA, ocmem icc is optional */
590 if (ret != -ENODATA)
591 goto fail;
592 ocmem_icc_path = NULL;
593 }
594
595
596 /*
597 * Set the ICC path to maximum speed for now by multiplying the fastest
598 * frequency by the bus width (8). We'll want to scale this later on to
599 * improve battery life.
600 */
601 icc_set_bw(icc_path, 0, Bps_to_icc(gpu->fast_rate) * 8);
602 icc_set_bw(ocmem_icc_path, 0, Bps_to_icc(gpu->fast_rate) * 8);
603
604 return gpu;
605
606 fail:
607 if (a3xx_gpu)
608 a3xx_destroy(&a3xx_gpu->base.base);
609
610 return ERR_PTR(ret);
611 }
612
613 const struct adreno_gpu_funcs a3xx_gpu_funcs = {
614 .base = {
615 .get_param = adreno_get_param,
616 .set_param = adreno_set_param,
617 .hw_init = a3xx_hw_init,
618 .pm_suspend = a3xx_pm_suspend,
619 .pm_resume = msm_gpu_pm_resume,
620 .recover = a3xx_recover,
621 .submit = a3xx_submit,
622 .active_ring = adreno_active_ring,
623 .irq = a3xx_irq,
624 .destroy = a3xx_destroy,
625 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
626 .show = adreno_show,
627 #endif
628 .gpu_busy = a3xx_gpu_busy,
629 .gpu_state_get = a3xx_gpu_state_get,
630 .gpu_state_put = adreno_gpu_state_put,
631 .create_vm = adreno_create_vm,
632 .get_rptr = a3xx_get_rptr,
633 },
634 .init = a3xx_gpu_init,
635 };
636