1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include <linux/kernel.h> 6 #include <linux/types.h> 7 #include <linux/cpumask.h> 8 #include <linux/firmware/qcom/qcom_pas.h> 9 #include <linux/pm_opp.h> 10 #include <linux/nvmem-consumer.h> 11 #include <linux/slab.h> 12 #include "msm_gem.h" 13 #include "msm_mmu.h" 14 #include "a5xx_gpu.h" 15 16 extern bool hang_debug; 17 static void a5xx_dump(struct msm_gpu *gpu); 18 19 #define GPU_PAS_ID 13 20 21 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 22 { 23 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 24 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 25 26 if (a5xx_gpu->has_whereami) { 27 OUT_PKT7(ring, CP_WHERE_AM_I, 2); 28 OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring))); 29 OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring))); 30 } 31 } 32 33 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, 34 bool sync) 35 { 36 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 37 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 38 uint32_t wptr; 39 unsigned long flags; 40 41 /* 42 * Most flush operations need to issue a WHERE_AM_I opcode to sync up 43 * the rptr shadow 44 */ 45 if (sync) 46 update_shadow_rptr(gpu, ring); 47 48 spin_lock_irqsave(&ring->preempt_lock, flags); 49 50 /* Copy the shadow to the actual register */ 51 ring->cur = ring->next; 52 53 /* Make sure to wrap wptr if we need to */ 54 wptr = get_wptr(ring); 55 56 spin_unlock_irqrestore(&ring->preempt_lock, flags); 57 58 /* Make sure everything is posted before making a decision */ 59 mb(); 60 61 /* Update HW if this is the current ring and we are not in preempt */ 62 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu)) 63 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 64 } 65 66 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit) 67 { 68 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 69 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 70 struct msm_ringbuffer *ring = submit->ring; 71 struct drm_gem_object *obj; 72 uint32_t *ptr, dwords; 73 unsigned int i; 74 75 for (i = 0; i < submit->nr_cmds; i++) { 76 switch (submit->cmd[i].type) { 77 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 78 break; 79 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 80 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno) 81 break; 82 fallthrough; 83 case MSM_SUBMIT_CMD_BUF: 84 /* copy commands into RB: */ 85 obj = submit->bos[submit->cmd[i].idx].obj; 86 dwords = submit->cmd[i].size; 87 88 ptr = msm_gem_get_vaddr(obj); 89 90 /* _get_vaddr() shouldn't fail at this point, 91 * since we've already mapped it once in 92 * submit_reloc() 93 */ 94 if (WARN_ON(IS_ERR_OR_NULL(ptr))) 95 return; 96 97 for (i = 0; i < dwords; i++) { 98 /* normally the OUT_PKTn() would wait 99 * for space for the packet. But since 100 * we just OUT_RING() the whole thing, 101 * need to call adreno_wait_ring() 102 * ourself: 103 */ 104 adreno_wait_ring(ring, 1); 105 OUT_RING(ring, ptr[i]); 106 } 107 108 msm_gem_put_vaddr(obj); 109 110 break; 111 } 112 } 113 114 a5xx_gpu->last_seqno[ring->id] = submit->seqno; 115 a5xx_flush(gpu, ring, true); 116 a5xx_preempt_trigger(gpu); 117 118 /* we might not necessarily have a cmd from userspace to 119 * trigger an event to know that submit has completed, so 120 * do this manually: 121 */ 122 a5xx_idle(gpu, ring); 123 ring->memptrs->fence = submit->seqno; 124 msm_gpu_retire(gpu); 125 } 126 127 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) 128 { 129 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 130 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 131 struct msm_ringbuffer *ring = submit->ring; 132 unsigned int i, ibs = 0; 133 134 adreno_check_and_reenable_stall(adreno_gpu); 135 136 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) { 137 ring->cur_ctx_seqno = 0; 138 a5xx_submit_in_rb(gpu, submit); 139 return; 140 } 141 142 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 143 OUT_RING(ring, 0x02); 144 145 /* Turn off protected mode to write to special registers */ 146 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 147 OUT_RING(ring, 0); 148 149 /* Set the save preemption record for the ring/command */ 150 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 151 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 152 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); 153 154 /* Turn back on protected mode */ 155 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 156 OUT_RING(ring, 1); 157 158 /* 159 * Disable local preemption by default because it requires 160 * user-space to be aware of it and provide additional handling 161 * to restore rendering state or do various flushes on switch. 162 */ 163 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 164 OUT_RING(ring, 0x0); 165 166 /* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */ 167 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 168 OUT_RING(ring, 0x02); 169 170 /* Submit the commands */ 171 for (i = 0; i < submit->nr_cmds; i++) { 172 switch (submit->cmd[i].type) { 173 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 174 break; 175 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 176 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno) 177 break; 178 fallthrough; 179 case MSM_SUBMIT_CMD_BUF: 180 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); 181 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); 182 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); 183 OUT_RING(ring, submit->cmd[i].size); 184 ibs++; 185 break; 186 } 187 188 /* 189 * Periodically update shadow-wptr if needed, so that we 190 * can see partial progress of submits with large # of 191 * cmds.. otherwise we could needlessly stall waiting for 192 * ringbuffer state, simply due to looking at a shadow 193 * rptr value that has not been updated 194 */ 195 if ((ibs % 32) == 0) 196 update_shadow_rptr(gpu, ring); 197 } 198 199 /* 200 * Write the render mode to NULL (0) to indicate to the CP that the IBs 201 * are done rendering - otherwise a lucky preemption would start 202 * replaying from the last checkpoint 203 */ 204 OUT_PKT7(ring, CP_SET_RENDER_MODE, 5); 205 OUT_RING(ring, 0); 206 OUT_RING(ring, 0); 207 OUT_RING(ring, 0); 208 OUT_RING(ring, 0); 209 OUT_RING(ring, 0); 210 211 /* Turn off IB level preemptions */ 212 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 213 OUT_RING(ring, 0x01); 214 215 /* Write the fence to the scratch register */ 216 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1); 217 OUT_RING(ring, submit->seqno); 218 a5xx_gpu->last_seqno[ring->id] = submit->seqno; 219 220 /* 221 * Execute a CACHE_FLUSH_TS event. This will ensure that the 222 * timestamp is written to the memory and then triggers the interrupt 223 */ 224 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 225 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) | 226 CP_EVENT_WRITE_0_IRQ); 227 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); 228 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); 229 OUT_RING(ring, submit->seqno); 230 231 /* Yield the floor on command completion */ 232 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 233 /* 234 * If dword[2:1] are non zero, they specify an address for the CP to 235 * write the value of dword[3] to on preemption complete. Write 0 to 236 * skip the write 237 */ 238 OUT_RING(ring, 0x00); 239 OUT_RING(ring, 0x00); 240 /* Data value - not used if the address above is 0 */ 241 OUT_RING(ring, 0x01); 242 /* Set bit 0 to trigger an interrupt on preempt complete */ 243 OUT_RING(ring, 0x01); 244 245 /* A WHERE_AM_I packet is not needed after a YIELD */ 246 a5xx_flush(gpu, ring, false); 247 248 /* Check to see if we need to start preemption */ 249 a5xx_preempt_trigger(gpu); 250 } 251 252 static const struct adreno_five_hwcg_regs { 253 u32 offset; 254 u32 value; 255 } a5xx_hwcg[] = { 256 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 257 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 258 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, 259 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, 260 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 261 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 262 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, 263 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, 264 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 265 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 266 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, 267 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, 268 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 269 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 270 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, 271 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, 272 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 273 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 274 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, 275 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, 276 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 277 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 278 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, 279 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, 280 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 281 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 282 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, 283 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, 284 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 285 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 286 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, 287 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, 288 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 289 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 290 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, 291 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, 292 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 293 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 294 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, 295 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, 296 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 297 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 298 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, 299 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, 300 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 301 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 302 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, 303 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, 304 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 305 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 306 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, 307 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, 308 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 309 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 310 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 311 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 312 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 313 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 314 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 315 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 316 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, 317 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, 318 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 319 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 320 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, 321 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, 322 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 323 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 324 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, 325 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, 326 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 327 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 328 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 329 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 330 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, 331 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, 332 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 333 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 334 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 335 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, 336 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, 337 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 338 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 339 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 340 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 341 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 342 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 343 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 344 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 345 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 346 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 347 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} 348 }, a50x_hwcg[] = { 349 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 350 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 351 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 352 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 353 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 354 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 355 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 356 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 357 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 358 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 359 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 360 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 361 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 362 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 363 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 364 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 365 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 366 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, 367 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 368 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 369 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 370 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 371 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 372 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 373 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 374 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 375 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 376 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 377 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 378 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 379 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 380 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 381 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 382 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 383 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 384 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 385 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 386 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 387 }, a512_hwcg[] = { 388 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, 389 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, 390 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, 391 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, 392 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, 393 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, 394 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, 395 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, 396 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, 397 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, 398 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, 399 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, 400 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, 401 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, 402 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, 403 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, 404 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, 405 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, 406 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, 407 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, 408 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, 409 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, 410 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, 411 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, 412 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, 413 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, 414 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, 415 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, 416 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, 417 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, 418 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, 419 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, 420 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, 421 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, 422 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, 423 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, 424 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, 425 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, 426 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, 427 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, 428 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, 429 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, 430 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, 431 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, 432 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, 433 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, 434 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, 435 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, 436 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, 437 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, 438 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, 439 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, 440 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, 441 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, 442 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, 443 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, 444 }; 445 446 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state) 447 { 448 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 449 const struct adreno_five_hwcg_regs *regs; 450 unsigned int i, sz; 451 452 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 453 adreno_is_a508(adreno_gpu)) { 454 regs = a50x_hwcg; 455 sz = ARRAY_SIZE(a50x_hwcg); 456 } else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) { 457 regs = a512_hwcg; 458 sz = ARRAY_SIZE(a512_hwcg); 459 } else { 460 regs = a5xx_hwcg; 461 sz = ARRAY_SIZE(a5xx_hwcg); 462 } 463 464 for (i = 0; i < sz; i++) 465 gpu_write(gpu, regs[i].offset, 466 state ? regs[i].value : 0); 467 468 if (adreno_is_a540(adreno_gpu)) { 469 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0); 470 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0); 471 } 472 473 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0); 474 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180); 475 } 476 477 static int a5xx_me_init(struct msm_gpu *gpu) 478 { 479 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 480 struct msm_ringbuffer *ring = gpu->rb[0]; 481 482 OUT_PKT7(ring, CP_ME_INIT, 8); 483 484 OUT_RING(ring, 0x0000002F); 485 486 /* Enable multiple hardware contexts */ 487 OUT_RING(ring, 0x00000003); 488 489 /* Enable error detection */ 490 OUT_RING(ring, 0x20000000); 491 492 /* Don't enable header dump */ 493 OUT_RING(ring, 0x00000000); 494 OUT_RING(ring, 0x00000000); 495 496 /* Specify workarounds for various microcode issues */ 497 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 498 adreno_is_a530(adreno_gpu)) { 499 /* Workaround for token end syncs 500 * Force a WFI after every direct-render 3D mode draw and every 501 * 2D mode 3 draw 502 */ 503 OUT_RING(ring, 0x0000000B); 504 } else if (adreno_is_a510(adreno_gpu)) { 505 /* Workaround for token and syncs */ 506 OUT_RING(ring, 0x00000001); 507 } else { 508 /* No workarounds enabled */ 509 OUT_RING(ring, 0x00000000); 510 } 511 512 OUT_RING(ring, 0x00000000); 513 OUT_RING(ring, 0x00000000); 514 515 a5xx_flush(gpu, ring, true); 516 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 517 } 518 519 static int a5xx_preempt_start(struct msm_gpu *gpu) 520 { 521 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 522 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 523 struct msm_ringbuffer *ring = gpu->rb[0]; 524 525 if (gpu->nr_rings == 1) 526 return 0; 527 528 /* Turn off protected mode to write to special registers */ 529 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 530 OUT_RING(ring, 0); 531 532 /* Set the save preemption record for the ring/command */ 533 OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2); 534 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id])); 535 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id])); 536 537 /* Turn back on protected mode */ 538 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 539 OUT_RING(ring, 1); 540 541 OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1); 542 OUT_RING(ring, 0x00); 543 544 OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1); 545 OUT_RING(ring, 0x01); 546 547 OUT_PKT7(ring, CP_YIELD_ENABLE, 1); 548 OUT_RING(ring, 0x01); 549 550 /* Yield the floor on command completion */ 551 OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 552 OUT_RING(ring, 0x00); 553 OUT_RING(ring, 0x00); 554 OUT_RING(ring, 0x01); 555 OUT_RING(ring, 0x01); 556 557 /* The WHERE_AMI_I packet is not needed after a YIELD is issued */ 558 a5xx_flush(gpu, ring, false); 559 560 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; 561 } 562 563 static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu, 564 struct drm_gem_object *obj) 565 { 566 u32 *buf = msm_gem_get_vaddr(obj); 567 568 if (IS_ERR(buf)) 569 return; 570 571 /* 572 * If the lowest nibble is 0xa that is an indication that this microcode 573 * has been patched. The actual version is in dword [3] but we only care 574 * about the patchlevel which is the lowest nibble of dword [3] 575 */ 576 if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) 577 a5xx_gpu->has_whereami = true; 578 579 msm_gem_put_vaddr(obj); 580 } 581 582 static int a5xx_ucode_load(struct msm_gpu *gpu) 583 { 584 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 585 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 586 int ret; 587 588 if (!a5xx_gpu->pm4_bo) { 589 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu, 590 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova); 591 592 593 if (IS_ERR(a5xx_gpu->pm4_bo)) { 594 ret = PTR_ERR(a5xx_gpu->pm4_bo); 595 a5xx_gpu->pm4_bo = NULL; 596 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n", 597 ret); 598 return ret; 599 } 600 601 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw"); 602 } 603 604 if (!a5xx_gpu->pfp_bo) { 605 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu, 606 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova); 607 608 if (IS_ERR(a5xx_gpu->pfp_bo)) { 609 ret = PTR_ERR(a5xx_gpu->pfp_bo); 610 a5xx_gpu->pfp_bo = NULL; 611 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n", 612 ret); 613 return ret; 614 } 615 616 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw"); 617 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo); 618 } 619 620 if (a5xx_gpu->has_whereami) { 621 if (!a5xx_gpu->shadow_bo) { 622 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, 623 sizeof(u32) * gpu->nr_rings, 624 MSM_BO_WC | MSM_BO_MAP_PRIV, 625 gpu->vm, &a5xx_gpu->shadow_bo, 626 &a5xx_gpu->shadow_iova); 627 628 if (IS_ERR(a5xx_gpu->shadow)) 629 return PTR_ERR(a5xx_gpu->shadow); 630 631 msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow"); 632 } 633 } else if (gpu->nr_rings > 1) { 634 /* Disable preemption if WHERE_AM_I isn't available */ 635 a5xx_preempt_fini(gpu); 636 gpu->nr_rings = 1; 637 } 638 639 return 0; 640 } 641 642 #define SCM_GPU_ZAP_SHADER_RESUME 0 643 644 static int a5xx_zap_shader_resume(struct msm_gpu *gpu) 645 { 646 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 647 int ret; 648 649 /* 650 * Adreno 506 have CPZ Retention feature and doesn't require 651 * to resume zap shader 652 */ 653 if (adreno_is_a506(adreno_gpu)) 654 return 0; 655 656 ret = qcom_pas_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID); 657 if (ret) 658 DRM_ERROR("%s: zap-shader resume failed: %d\n", 659 gpu->name, ret); 660 661 return ret; 662 } 663 664 static int a5xx_zap_shader_init(struct msm_gpu *gpu) 665 { 666 static bool loaded; 667 int ret; 668 669 /* 670 * If the zap shader is already loaded into memory we just need to kick 671 * the remote processor to reinitialize it 672 */ 673 if (loaded) 674 return a5xx_zap_shader_resume(gpu); 675 676 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID); 677 678 loaded = !ret; 679 return ret; 680 } 681 682 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 683 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 684 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 685 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 686 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 687 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \ 688 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 689 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \ 690 A5XX_RBBM_INT_0_MASK_CP_SW | \ 691 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 692 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 693 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 694 695 static int a5xx_hw_init(struct msm_gpu *gpu) 696 { 697 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 698 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 699 u32 hbb; 700 int ret; 701 702 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); 703 704 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 705 adreno_is_a540(adreno_gpu)) 706 gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); 707 708 /* Make all blocks contribute to the GPU BUSY perf counter */ 709 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); 710 711 /* Enable RBBM error reporting bits */ 712 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001); 713 714 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) { 715 /* 716 * Mask out the activity signals from RB1-3 to avoid false 717 * positives 718 */ 719 720 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, 721 0xF0000000); 722 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, 723 0xFFFFFFFF); 724 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, 725 0xFFFFFFFF); 726 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, 727 0xFFFFFFFF); 728 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, 729 0xFFFFFFFF); 730 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, 731 0xFFFFFFFF); 732 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, 733 0xFFFFFFFF); 734 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, 735 0xFFFFFFFF); 736 } 737 738 /* Enable fault detection */ 739 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL, 740 (1 << 30) | 0xFFFF); 741 742 /* Turn on performance counters */ 743 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01); 744 745 /* Select CP0 to always count cycles */ 746 gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); 747 748 /* Select RBBM0 to countable 6 to get the busy status for devfreq */ 749 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); 750 751 /* Increase VFD cache access so LRZ and other data gets evicted less */ 752 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02); 753 754 /* Disable L2 bypass in the UCHE */ 755 gpu_write64(gpu, REG_A5XX_UCHE_TRAP_BASE, adreno_gpu->uche_trap_base); 756 gpu_write64(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base); 757 758 /* Set the GMEM VA range (0 to gpu->gmem) */ 759 gpu_write64(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN, 0x00100000); 760 gpu_write64(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX, 761 0x00100000 + adreno_gpu->info->gmem - 1); 762 763 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 764 adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) { 765 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20); 766 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 767 adreno_is_a508(adreno_gpu)) 768 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 769 else 770 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20); 771 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); 772 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); 773 } else { 774 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40); 775 if (adreno_is_a530(adreno_gpu)) 776 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40); 777 else 778 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400); 779 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); 780 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); 781 } 782 783 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 784 adreno_is_a508(adreno_gpu)) 785 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 786 (0x100 << 11 | 0x100 << 22)); 787 else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) || 788 adreno_is_a512(adreno_gpu)) 789 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 790 (0x200 << 11 | 0x200 << 22)); 791 else 792 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 793 (0x400 << 11 | 0x300 << 22)); 794 795 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) 796 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); 797 798 /* 799 * Disable the RB sampler datapath DP2 clock gating optimization 800 * for 1-SP GPUs, as it is enabled by default. 801 */ 802 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 803 adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) || 804 adreno_is_a512(adreno_gpu)) 805 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9)); 806 807 /* Disable UCHE global filter as SP can invalidate/flush independently */ 808 gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29)); 809 810 /* Enable USE_RETENTION_FLOPS */ 811 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000); 812 813 /* Enable ME/PFP split notification */ 814 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); 815 816 /* 817 * In A5x, CCU can send context_done event of a particular context to 818 * UCHE which ultimately reaches CP even when there is valid 819 * transaction of that context inside CCU. This can let CP to program 820 * config registers, which will make the "valid transaction" inside 821 * CCU to be interpreted differently. This can cause gpu fault. This 822 * bug is fixed in latest A510 revision. To enable this bug fix - 823 * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1 824 * (disable). For older A510 version this bit is unused. 825 */ 826 if (adreno_is_a510(adreno_gpu)) 827 gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0); 828 829 /* Enable HWCG */ 830 a5xx_set_hwcg(gpu, true); 831 832 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F); 833 834 BUG_ON(adreno_gpu->ubwc_config->highest_bank_bit < 13); 835 hbb = adreno_gpu->ubwc_config->highest_bank_bit - 13; 836 837 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, hbb << 7); 838 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, hbb << 1); 839 840 if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) || 841 adreno_is_a540(adreno_gpu)) 842 gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, hbb); 843 844 /* Disable All flat shading optimization (ALLFLATOPTDIS) */ 845 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10)); 846 847 /* Protect registers from the CP */ 848 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007); 849 850 /* RBBM */ 851 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4)); 852 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8)); 853 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16)); 854 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32)); 855 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64)); 856 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64)); 857 858 /* Content protect */ 859 gpu_write(gpu, REG_A5XX_CP_PROTECT(6), 860 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 861 16)); 862 gpu_write(gpu, REG_A5XX_CP_PROTECT(7), 863 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2)); 864 865 /* CP */ 866 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64)); 867 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8)); 868 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32)); 869 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1)); 870 871 /* RB */ 872 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1)); 873 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2)); 874 875 /* VPC */ 876 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8)); 877 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16)); 878 879 /* UCHE */ 880 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16)); 881 882 /* SMMU */ 883 gpu_write(gpu, REG_A5XX_CP_PROTECT(17), 884 ADRENO_PROTECT_RW(0x10000, 0x8000)); 885 886 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0); 887 /* 888 * Disable the trusted memory range - we don't actually supported secure 889 * memory rendering at this point in time and we don't want to block off 890 * part of the virtual memory space. 891 */ 892 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000); 893 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 894 895 /* Put the GPU into 64 bit by default */ 896 gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1); 897 gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1); 898 gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1); 899 gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1); 900 gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1); 901 gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); 902 gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1); 903 gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1); 904 gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1); 905 gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1); 906 gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1); 907 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); 908 909 /* 910 * VPC corner case with local memory load kill leads to corrupt 911 * internal state. Normal Disable does not work for all a5x chips. 912 * So do the following setting to disable it. 913 */ 914 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) { 915 gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23)); 916 gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0); 917 } 918 919 ret = adreno_hw_init(gpu); 920 if (ret) 921 return ret; 922 923 if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu)) 924 a5xx_gpmu_ucode_init(gpu); 925 926 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova); 927 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova); 928 929 /* Set the ringbuffer address */ 930 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova); 931 932 /* 933 * If the microcode supports the WHERE_AM_I opcode then we can use that 934 * in lieu of the RPTR shadow and enable preemption. Otherwise, we 935 * can't safely use the RPTR shadow or preemption. In either case, the 936 * RPTR shadow should be disabled in hardware. 937 */ 938 gpu_write(gpu, REG_A5XX_CP_RB_CNTL, 939 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE); 940 941 /* Configure the RPTR shadow if needed: */ 942 if (a5xx_gpu->shadow_bo) { 943 gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR, 944 shadowptr(a5xx_gpu, gpu->rb[0])); 945 } 946 947 a5xx_preempt_hw_init(gpu); 948 949 /* Disable the interrupts through the initial bringup stage */ 950 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK); 951 952 /* Clear ME_HALT to start the micro engine */ 953 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0); 954 ret = a5xx_me_init(gpu); 955 if (ret) 956 return ret; 957 958 ret = a5xx_power_init(gpu); 959 if (ret) 960 return ret; 961 962 /* 963 * Send a pipeline event stat to get misbehaving counters to start 964 * ticking correctly 965 */ 966 if (adreno_is_a530(adreno_gpu)) { 967 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1); 968 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT)); 969 970 a5xx_flush(gpu, gpu->rb[0], true); 971 if (!a5xx_idle(gpu, gpu->rb[0])) 972 return -EINVAL; 973 } 974 975 /* 976 * If the chip that we are using does support loading one, then 977 * try to load a zap shader into the secure world. If successful 978 * we can use the CP to switch out of secure mode. If not then we 979 * have no resource but to try to switch ourselves out manually. If we 980 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will 981 * be blocked and a permissions violation will soon follow. 982 */ 983 ret = a5xx_zap_shader_init(gpu); 984 if (!ret) { 985 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); 986 OUT_RING(gpu->rb[0], 0x00000000); 987 988 a5xx_flush(gpu, gpu->rb[0], true); 989 if (!a5xx_idle(gpu, gpu->rb[0])) 990 return -EINVAL; 991 } else if (ret == -ENODEV) { 992 /* 993 * This device does not use zap shader (but print a warning 994 * just in case someone got their dt wrong.. hopefully they 995 * have a debug UART to realize the error of their ways... 996 * if you mess this up you are about to crash horribly) 997 */ 998 dev_warn_once(gpu->dev->dev, 999 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); 1000 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); 1001 } else { 1002 return ret; 1003 } 1004 1005 /* Last step - yield the ringbuffer */ 1006 a5xx_preempt_start(gpu); 1007 1008 return 0; 1009 } 1010 1011 static void a5xx_recover(struct msm_gpu *gpu) 1012 { 1013 int i; 1014 1015 adreno_dump_info(gpu); 1016 1017 for (i = 0; i < 8; i++) { 1018 printk("CP_SCRATCH_REG%d: %u\n", i, 1019 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i))); 1020 } 1021 1022 if (hang_debug) 1023 a5xx_dump(gpu); 1024 1025 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1); 1026 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD); 1027 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0); 1028 adreno_recover(gpu); 1029 } 1030 1031 static void a5xx_destroy(struct msm_gpu *gpu) 1032 { 1033 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1034 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1035 1036 DBG("%s", gpu->name); 1037 1038 a5xx_preempt_fini(gpu); 1039 1040 if (a5xx_gpu->pm4_bo) { 1041 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->vm); 1042 drm_gem_object_put(a5xx_gpu->pm4_bo); 1043 } 1044 1045 if (a5xx_gpu->pfp_bo) { 1046 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->vm); 1047 drm_gem_object_put(a5xx_gpu->pfp_bo); 1048 } 1049 1050 if (a5xx_gpu->gpmu_bo) { 1051 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->vm); 1052 drm_gem_object_put(a5xx_gpu->gpmu_bo); 1053 } 1054 1055 if (a5xx_gpu->shadow_bo) { 1056 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->vm); 1057 drm_gem_object_put(a5xx_gpu->shadow_bo); 1058 } 1059 1060 adreno_gpu_cleanup(adreno_gpu); 1061 kfree(a5xx_gpu); 1062 } 1063 1064 static inline bool _a5xx_check_idle(struct msm_gpu *gpu) 1065 { 1066 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY) 1067 return false; 1068 1069 /* 1070 * Nearly every abnormality ends up pausing the GPU and triggering a 1071 * fault so we can safely just watch for this one interrupt to fire 1072 */ 1073 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) & 1074 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT); 1075 } 1076 1077 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1078 { 1079 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1080 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1081 1082 if (ring != a5xx_gpu->cur_ring) { 1083 WARN(1, "Tried to idle a non-current ringbuffer\n"); 1084 return false; 1085 } 1086 1087 /* wait for CP to drain ringbuffer: */ 1088 if (!adreno_idle(gpu, ring)) 1089 return false; 1090 1091 if (spin_until(_a5xx_check_idle(gpu))) { 1092 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 1093 gpu->name, __builtin_return_address(0), 1094 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1095 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS), 1096 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1097 gpu_read(gpu, REG_A5XX_CP_RB_WPTR)); 1098 return false; 1099 } 1100 1101 return true; 1102 } 1103 1104 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data) 1105 { 1106 struct msm_gpu *gpu = arg; 1107 struct adreno_smmu_fault_info *info = data; 1108 char block[12] = "unknown"; 1109 u32 scratch[] = { 1110 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)), 1111 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)), 1112 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)), 1113 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)), 1114 }; 1115 1116 if (info) 1117 snprintf(block, sizeof(block), "%x", info->fsynr1); 1118 1119 return adreno_fault_handler(gpu, iova, flags, info, block, scratch); 1120 } 1121 1122 static void a5xx_cp_err_irq(struct msm_gpu *gpu) 1123 { 1124 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS); 1125 1126 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) { 1127 u32 val; 1128 1129 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0); 1130 1131 /* 1132 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so 1133 * read it twice 1134 */ 1135 1136 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1137 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA); 1138 1139 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n", 1140 val); 1141 } 1142 1143 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR) 1144 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n", 1145 gpu_read(gpu, REG_A5XX_CP_HW_FAULT)); 1146 1147 if (status & A5XX_CP_INT_CP_DMA_ERROR) 1148 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n"); 1149 1150 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { 1151 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS); 1152 1153 dev_err_ratelimited(gpu->dev->dev, 1154 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", 1155 val & (1 << 24) ? "WRITE" : "READ", 1156 (val & 0xFFFFF) >> 2, val); 1157 } 1158 1159 if (status & A5XX_CP_INT_CP_AHB_ERROR) { 1160 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT); 1161 const char *access[16] = { "reserved", "reserved", 1162 "timestamp lo", "timestamp hi", "pfp read", "pfp write", 1163 "", "", "me read", "me write", "", "", "crashdump read", 1164 "crashdump write" }; 1165 1166 dev_err_ratelimited(gpu->dev->dev, 1167 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n", 1168 status & 0xFFFFF, access[(status >> 24) & 0xF], 1169 (status & (1 << 31)), status); 1170 } 1171 } 1172 1173 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status) 1174 { 1175 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) { 1176 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS); 1177 1178 dev_err_ratelimited(gpu->dev->dev, 1179 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n", 1180 val & (1 << 28) ? "WRITE" : "READ", 1181 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3, 1182 (val >> 24) & 0xF); 1183 1184 /* Clear the error */ 1185 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4)); 1186 1187 /* Clear the interrupt */ 1188 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1189 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1190 } 1191 1192 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT) 1193 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n"); 1194 1195 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT) 1196 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n", 1197 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS)); 1198 1199 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT) 1200 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n", 1201 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS)); 1202 1203 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT) 1204 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n", 1205 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS)); 1206 1207 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1208 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n"); 1209 1210 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 1211 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n"); 1212 } 1213 1214 static void a5xx_uche_err_irq(struct msm_gpu *gpu) 1215 { 1216 uint64_t addr = gpu_read64(gpu, REG_A5XX_UCHE_TRAP_LOG); 1217 1218 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n", 1219 addr); 1220 } 1221 1222 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu) 1223 { 1224 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n"); 1225 } 1226 1227 static void a5xx_fault_detect_irq(struct msm_gpu *gpu) 1228 { 1229 struct drm_device *dev = gpu->dev; 1230 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 1231 1232 /* 1233 * If stalled on SMMU fault, we could trip the GPU's hang detection, 1234 * but the fault handler will trigger the devcore dump, and we want 1235 * to otherwise resume normally rather than killing the submit, so 1236 * just bail. 1237 */ 1238 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)) 1239 return; 1240 1241 DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", 1242 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, 1243 gpu_read(gpu, REG_A5XX_RBBM_STATUS), 1244 gpu_read(gpu, REG_A5XX_CP_RB_RPTR), 1245 gpu_read(gpu, REG_A5XX_CP_RB_WPTR), 1246 gpu_read64(gpu, REG_A5XX_CP_IB1_BASE), 1247 gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ), 1248 gpu_read64(gpu, REG_A5XX_CP_IB2_BASE), 1249 gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ)); 1250 1251 /* Turn off the hangcheck timer to keep it from bothering us */ 1252 timer_delete(&gpu->hangcheck_timer); 1253 1254 kthread_queue_work(gpu->worker, &gpu->recover_work); 1255 } 1256 1257 #define RBBM_ERROR_MASK \ 1258 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \ 1259 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \ 1260 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \ 1261 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \ 1262 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \ 1263 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW) 1264 1265 static irqreturn_t a5xx_irq(struct msm_gpu *gpu) 1266 { 1267 struct msm_drm_private *priv = gpu->dev->dev_private; 1268 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS); 1269 1270 /* 1271 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it 1272 * before the source is cleared the interrupt will storm. 1273 */ 1274 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, 1275 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); 1276 1277 if (priv->disable_err_irq) { 1278 /* Turn off interrupts to avoid interrupt storm */ 1279 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, 1280 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | 1281 A5XX_RBBM_INT_0_MASK_CP_SW); 1282 1283 status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | 1284 A5XX_RBBM_INT_0_MASK_CP_SW; 1285 } 1286 1287 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */ 1288 if (status & RBBM_ERROR_MASK) 1289 a5xx_rbbm_err_irq(gpu, status); 1290 1291 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR) 1292 a5xx_cp_err_irq(gpu); 1293 1294 if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT) 1295 a5xx_fault_detect_irq(gpu); 1296 1297 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 1298 a5xx_uche_err_irq(gpu); 1299 1300 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP) 1301 a5xx_gpmu_err_irq(gpu); 1302 1303 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1304 a5xx_preempt_trigger(gpu); 1305 msm_gpu_retire(gpu); 1306 } 1307 1308 if (status & A5XX_RBBM_INT_0_MASK_CP_SW) 1309 a5xx_preempt_irq(gpu); 1310 1311 return IRQ_HANDLED; 1312 } 1313 1314 static const u32 a5xx_registers[] = { 1315 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, 1316 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, 1317 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, 1318 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841, 1319 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28, 1320 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 1321 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 1322 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, 1323 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 1324 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 1325 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, 1326 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, 1327 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 1328 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 1329 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 1330 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 1331 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A, 1332 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F, 1333 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 1334 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947, 1335 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 1336 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68, 1337 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 1338 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 1339 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 1340 0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D, 1341 0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5, 1342 0xAC60, 0xAC60, ~0, 1343 }; 1344 1345 static void a5xx_dump(struct msm_gpu *gpu) 1346 { 1347 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n", 1348 gpu_read(gpu, REG_A5XX_RBBM_STATUS)); 1349 adreno_dump(gpu); 1350 } 1351 1352 static int a5xx_pm_resume(struct msm_gpu *gpu) 1353 { 1354 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1355 int ret; 1356 1357 /* Turn on the core power */ 1358 ret = msm_gpu_pm_resume(gpu); 1359 if (ret) 1360 return ret; 1361 1362 /* Adreno 505, 506, 508, 509, 510, 512 needs manual RBBM sus/res control */ 1363 if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) { 1364 /* Halt the sp_input_clk at HM level */ 1365 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055); 1366 a5xx_set_hwcg(gpu, true); 1367 /* Turn on sp_input_clk at HM level */ 1368 gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0); 1369 return 0; 1370 } 1371 1372 /* Turn the RBCCU domain first to limit the chances of voltage droop */ 1373 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); 1374 1375 /* Wait 3 usecs before polling */ 1376 udelay(3); 1377 1378 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1379 (1 << 20), (1 << 20)); 1380 if (ret) { 1381 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n", 1382 gpu->name, 1383 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS)); 1384 return ret; 1385 } 1386 1387 /* Turn on the SP domain */ 1388 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000); 1389 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS, 1390 (1 << 20), (1 << 20)); 1391 if (ret) 1392 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n", 1393 gpu->name); 1394 1395 return ret; 1396 } 1397 1398 static int a5xx_pm_suspend(struct msm_gpu *gpu) 1399 { 1400 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1401 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1402 u32 mask = 0xf; 1403 int i, ret; 1404 1405 /* A505, A506, A508, A510 have 3 XIN ports in VBIF */ 1406 if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) || 1407 adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) 1408 mask = 0x7; 1409 1410 /* Clear the VBIF pipe before shutting down */ 1411 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask); 1412 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 1413 mask) == mask); 1414 1415 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0); 1416 1417 /* 1418 * Reset the VBIF before power collapse to avoid issue with FIFO 1419 * entries on Adreno A510 and A530 (the others will tend to lock up) 1420 */ 1421 if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) { 1422 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000); 1423 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000); 1424 } 1425 1426 ret = msm_gpu_pm_suspend(gpu); 1427 if (ret) 1428 return ret; 1429 1430 if (a5xx_gpu->has_whereami) 1431 for (i = 0; i < gpu->nr_rings; i++) 1432 a5xx_gpu->shadow[i] = 0; 1433 1434 return 0; 1435 } 1436 1437 static u64 a5xx_get_timestamp(struct msm_gpu *gpu) 1438 { 1439 return gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO); 1440 } 1441 1442 struct a5xx_crashdumper { 1443 void *ptr; 1444 struct drm_gem_object *bo; 1445 u64 iova; 1446 }; 1447 1448 struct a5xx_gpu_state { 1449 struct msm_gpu_state base; 1450 u32 *hlsqregs; 1451 }; 1452 1453 static int a5xx_crashdumper_init(struct msm_gpu *gpu, 1454 struct a5xx_crashdumper *dumper) 1455 { 1456 dumper->ptr = msm_gem_kernel_new(gpu->dev, 1457 SZ_1M, MSM_BO_WC, gpu->vm, 1458 &dumper->bo, &dumper->iova); 1459 1460 if (!IS_ERR(dumper->ptr)) 1461 msm_gem_object_set_name(dumper->bo, "crashdump"); 1462 1463 return PTR_ERR_OR_ZERO(dumper->ptr); 1464 } 1465 1466 static int a5xx_crashdumper_run(struct msm_gpu *gpu, 1467 struct a5xx_crashdumper *dumper) 1468 { 1469 u32 val; 1470 1471 if (IS_ERR_OR_NULL(dumper->ptr)) 1472 return -EINVAL; 1473 1474 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova); 1475 1476 gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1); 1477 1478 return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val, 1479 val & 0x04, 100, 10000); 1480 } 1481 1482 /* 1483 * These are a list of the registers that need to be read through the HLSQ 1484 * aperture through the crashdumper. These are not nominally accessible from 1485 * the CPU on a secure platform. 1486 */ 1487 static const struct { 1488 u32 type; 1489 u32 regoffset; 1490 u32 count; 1491 } a5xx_hlsq_aperture_regs[] = { 1492 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */ 1493 { 0x31, 0x2080, 0x1 }, /* HLSQ 2D context 0 */ 1494 { 0x33, 0x2480, 0x1 }, /* HLSQ 2D context 1 */ 1495 { 0x32, 0xe780, 0x62 }, /* HLSQ 3D context 0 */ 1496 { 0x34, 0xef80, 0x62 }, /* HLSQ 3D context 1 */ 1497 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */ 1498 { 0x3d, 0x2040, 0x1 }, /* SP 2D context 0 */ 1499 { 0x3b, 0x2440, 0x1 }, /* SP 2D context 1 */ 1500 { 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */ 1501 { 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */ 1502 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */ 1503 { 0x38, 0x2000, 0xa }, /* TP 2D context 0 */ 1504 { 0x36, 0x2400, 0xa }, /* TP 2D context 1 */ 1505 { 0x39, 0xe700, 0x80 }, /* TP 3D context 0 */ 1506 { 0x37, 0xef00, 0x80 }, /* TP 3D context 1 */ 1507 }; 1508 1509 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu, 1510 struct a5xx_gpu_state *a5xx_state) 1511 { 1512 struct a5xx_crashdumper dumper = { 0 }; 1513 u32 offset, count = 0; 1514 u64 *ptr; 1515 int i; 1516 1517 if (a5xx_crashdumper_init(gpu, &dumper)) 1518 return; 1519 1520 /* The script will be written at offset 0 */ 1521 ptr = dumper.ptr; 1522 1523 /* Start writing the data at offset 256k */ 1524 offset = dumper.iova + (256 * SZ_1K); 1525 1526 /* Count how many additional registers to get from the HLSQ aperture */ 1527 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) 1528 count += a5xx_hlsq_aperture_regs[i].count; 1529 1530 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL); 1531 if (!a5xx_state->hlsqregs) 1532 return; 1533 1534 /* Build the crashdump script */ 1535 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1536 u32 type = a5xx_hlsq_aperture_regs[i].type; 1537 u32 c = a5xx_hlsq_aperture_regs[i].count; 1538 1539 /* Write the register to select the desired bank */ 1540 *ptr++ = ((u64) type << 8); 1541 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) | 1542 (1 << 21) | 1; 1543 1544 *ptr++ = offset; 1545 *ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44) 1546 | c; 1547 1548 offset += c * sizeof(u32); 1549 } 1550 1551 /* Write two zeros to close off the script */ 1552 *ptr++ = 0; 1553 *ptr++ = 0; 1554 1555 if (a5xx_crashdumper_run(gpu, &dumper)) { 1556 kfree(a5xx_state->hlsqregs); 1557 msm_gem_kernel_put(dumper.bo, gpu->vm); 1558 return; 1559 } 1560 1561 /* Copy the data from the crashdumper to the state */ 1562 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K), 1563 count * sizeof(u32)); 1564 1565 msm_gem_kernel_put(dumper.bo, gpu->vm); 1566 } 1567 1568 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu) 1569 { 1570 struct a5xx_gpu_state *a5xx_state = kzalloc_obj(*a5xx_state); 1571 bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24)); 1572 1573 if (!a5xx_state) 1574 return ERR_PTR(-ENOMEM); 1575 1576 /* Temporarily disable hardware clock gating before reading the hw */ 1577 a5xx_set_hwcg(gpu, false); 1578 1579 /* First get the generic state from the adreno core */ 1580 adreno_gpu_state_get(gpu, &(a5xx_state->base)); 1581 1582 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS); 1583 1584 /* 1585 * Get the HLSQ regs with the help of the crashdumper, but only if 1586 * we are not stalled in an iommu fault (in which case the crashdumper 1587 * would not have access to memory) 1588 */ 1589 if (!stalled) 1590 a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state); 1591 1592 a5xx_set_hwcg(gpu, true); 1593 1594 return &a5xx_state->base; 1595 } 1596 1597 static void a5xx_gpu_state_destroy(struct kref *kref) 1598 { 1599 struct msm_gpu_state *state = container_of(kref, 1600 struct msm_gpu_state, ref); 1601 struct a5xx_gpu_state *a5xx_state = container_of(state, 1602 struct a5xx_gpu_state, base); 1603 1604 kfree(a5xx_state->hlsqregs); 1605 1606 adreno_gpu_state_destroy(state); 1607 kfree(a5xx_state); 1608 } 1609 1610 static int a5xx_gpu_state_put(struct msm_gpu_state *state) 1611 { 1612 if (IS_ERR_OR_NULL(state)) 1613 return 1; 1614 1615 return kref_put(&state->ref, a5xx_gpu_state_destroy); 1616 } 1617 1618 1619 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1620 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 1621 struct drm_printer *p) 1622 { 1623 int i, j; 1624 u32 pos = 0; 1625 struct a5xx_gpu_state *a5xx_state = container_of(state, 1626 struct a5xx_gpu_state, base); 1627 1628 if (IS_ERR_OR_NULL(state)) 1629 return; 1630 1631 adreno_show(gpu, state, p); 1632 1633 /* Dump the additional a5xx HLSQ registers */ 1634 if (!a5xx_state->hlsqregs) 1635 return; 1636 1637 drm_printf(p, "registers-hlsq:\n"); 1638 1639 for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) { 1640 u32 o = a5xx_hlsq_aperture_regs[i].regoffset; 1641 u32 c = a5xx_hlsq_aperture_regs[i].count; 1642 1643 for (j = 0; j < c; j++, pos++, o++) { 1644 /* 1645 * To keep the crashdump simple we pull the entire range 1646 * for each register type but not all of the registers 1647 * in the range are valid. Fortunately invalid registers 1648 * stick out like a sore thumb with a value of 1649 * 0xdeadbeef 1650 */ 1651 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef) 1652 continue; 1653 1654 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n", 1655 o << 2, a5xx_state->hlsqregs[pos]); 1656 } 1657 } 1658 } 1659 #endif 1660 1661 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu) 1662 { 1663 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1664 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1665 1666 return a5xx_gpu->cur_ring; 1667 } 1668 1669 static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) 1670 { 1671 u64 busy_cycles; 1672 1673 busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO); 1674 *out_sample_rate = clk_get_rate(gpu->core_clk); 1675 1676 return busy_cycles; 1677 } 1678 1679 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1680 { 1681 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1682 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 1683 1684 if (a5xx_gpu->has_whereami) 1685 return a5xx_gpu->shadow[ring->id]; 1686 1687 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR); 1688 } 1689 1690 static void check_speed_bin(struct device *dev) 1691 { 1692 struct nvmem_cell *cell; 1693 u32 val; 1694 1695 /* 1696 * If the OPP table specifies a opp-supported-hw property then we have 1697 * to set something with dev_pm_opp_set_supported_hw() or the table 1698 * doesn't get populated so pick an arbitrary value that should 1699 * ensure the default frequencies are selected but not conflict with any 1700 * actual bins 1701 */ 1702 val = 0x80; 1703 1704 cell = nvmem_cell_get(dev, "speed_bin"); 1705 1706 if (!IS_ERR(cell)) { 1707 void *buf = nvmem_cell_read(cell, NULL); 1708 1709 if (!IS_ERR(buf)) { 1710 u8 bin = *((u8 *) buf); 1711 1712 val = (1 << bin); 1713 kfree(buf); 1714 } 1715 1716 nvmem_cell_put(cell); 1717 } 1718 1719 devm_pm_opp_set_supported_hw(dev, &val, 1); 1720 } 1721 1722 static struct msm_gpu *a5xx_gpu_init(struct drm_device *dev) 1723 { 1724 struct msm_drm_private *priv = dev->dev_private; 1725 struct platform_device *pdev = priv->gpu_pdev; 1726 struct adreno_platform_config *config = pdev->dev.platform_data; 1727 struct a5xx_gpu *a5xx_gpu = NULL; 1728 struct adreno_gpu *adreno_gpu; 1729 struct msm_gpu *gpu; 1730 unsigned int nr_rings; 1731 u32 speedbin; 1732 int ret; 1733 1734 a5xx_gpu = kzalloc_obj(*a5xx_gpu); 1735 if (!a5xx_gpu) 1736 return ERR_PTR(-ENOMEM); 1737 1738 adreno_gpu = &a5xx_gpu->base; 1739 gpu = &adreno_gpu->base; 1740 1741 adreno_gpu->registers = a5xx_registers; 1742 1743 a5xx_gpu->lm_leakage = 0x4E001A; 1744 1745 check_speed_bin(&pdev->dev); 1746 1747 nr_rings = 4; 1748 1749 if (config->info->revn == 510) 1750 nr_rings = 1; 1751 1752 ret = adreno_gpu_init(dev, pdev, adreno_gpu, config->info->funcs, nr_rings); 1753 if (ret) { 1754 a5xx_destroy(&(a5xx_gpu->base.base)); 1755 return ERR_PTR(ret); 1756 } 1757 1758 /* Set the speedbin value that is passed to userspace */ 1759 if (adreno_read_speedbin(&pdev->dev, &speedbin) || !speedbin) 1760 speedbin = 0xffff; 1761 adreno_gpu->speedbin = (uint16_t) (0xffff & speedbin); 1762 1763 msm_mmu_set_fault_handler(to_msm_vm(gpu->vm)->mmu, gpu, 1764 a5xx_fault_handler); 1765 1766 /* Set up the preemption specific bits and pieces for each ringbuffer */ 1767 a5xx_preempt_init(gpu); 1768 1769 /* Inherit the common config and make some necessary fixups */ 1770 adreno_gpu->ubwc_config = qcom_ubwc_config_get_data(); 1771 if (IS_ERR(adreno_gpu->ubwc_config)) 1772 return ERR_CAST(adreno_gpu->ubwc_config); 1773 1774 adreno_gpu->uche_trap_base = 0x0001ffffffff0000ull; 1775 1776 return gpu; 1777 } 1778 1779 const struct adreno_gpu_funcs a5xx_gpu_funcs = { 1780 .base = { 1781 .get_param = adreno_get_param, 1782 .set_param = adreno_set_param, 1783 .hw_init = a5xx_hw_init, 1784 .ucode_load = a5xx_ucode_load, 1785 .pm_suspend = a5xx_pm_suspend, 1786 .pm_resume = a5xx_pm_resume, 1787 .recover = a5xx_recover, 1788 .submit = a5xx_submit, 1789 .active_ring = a5xx_active_ring, 1790 .irq = a5xx_irq, 1791 .destroy = a5xx_destroy, 1792 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) 1793 .show = a5xx_show, 1794 #endif 1795 #if defined(CONFIG_DEBUG_FS) 1796 .debugfs_init = a5xx_debugfs_init, 1797 #endif 1798 .gpu_busy = a5xx_gpu_busy, 1799 .gpu_state_get = a5xx_gpu_state_get, 1800 .gpu_state_put = a5xx_gpu_state_put, 1801 .create_vm = adreno_create_vm, 1802 .get_rptr = a5xx_get_rptr, 1803 }, 1804 .init = a5xx_gpu_init, 1805 .get_timestamp = a5xx_get_timestamp, 1806 }; 1807