1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ 3 4 5 #include "msm_gem.h" 6 #include "msm_mmu.h" 7 #include "msm_gpu_trace.h" 8 #include "a6xx_gpu.h" 9 #include "a6xx_gmu.xml.h" 10 11 #include <linux/bitfield.h> 12 #include <linux/devfreq.h> 13 #include <linux/firmware/qcom/qcom_scm.h> 14 #include <linux/pm_domain.h> 15 #include <linux/soc/qcom/llcc-qcom.h> 16 17 #define GPU_PAS_ID 13 18 19 static void a8xx_aperture_slice_set(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 slice) 20 { 21 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 22 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 23 u32 val; 24 25 val = A8XX_CP_APERTURE_CNTL_HOST_PIPEID(pipe) | A8XX_CP_APERTURE_CNTL_HOST_SLICEID(slice); 26 27 if (a6xx_gpu->cached_aperture == val) 28 return; 29 30 gpu_write(gpu, REG_A8XX_CP_APERTURE_CNTL_HOST, val); 31 32 a6xx_gpu->cached_aperture = val; 33 } 34 35 static void a8xx_aperture_acquire(struct msm_gpu *gpu, enum adreno_pipe pipe, unsigned long *flags) 36 { 37 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 38 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 39 40 spin_lock_irqsave(&a6xx_gpu->aperture_lock, *flags); 41 42 a8xx_aperture_slice_set(gpu, pipe, 0); 43 } 44 45 static void a8xx_aperture_release(struct msm_gpu *gpu, unsigned long flags) 46 { 47 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 48 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 49 50 spin_unlock_irqrestore(&a6xx_gpu->aperture_lock, flags); 51 } 52 53 static void a8xx_aperture_clear(struct msm_gpu *gpu) 54 { 55 unsigned long flags; 56 57 a8xx_aperture_acquire(gpu, PIPE_NONE, &flags); 58 a8xx_aperture_release(gpu, flags); 59 } 60 61 static void a8xx_write_pipe(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 offset, u32 data) 62 { 63 unsigned long flags; 64 65 a8xx_aperture_acquire(gpu, pipe, &flags); 66 gpu_write(gpu, offset, data); 67 a8xx_aperture_release(gpu, flags); 68 } 69 70 static u32 a8xx_read_pipe_slice(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 slice, u32 offset) 71 { 72 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 73 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 74 unsigned long flags; 75 u32 val; 76 77 spin_lock_irqsave(&a6xx_gpu->aperture_lock, flags); 78 a8xx_aperture_slice_set(gpu, pipe, slice); 79 val = gpu_read(gpu, offset); 80 spin_unlock_irqrestore(&a6xx_gpu->aperture_lock, flags); 81 82 return val; 83 } 84 85 void a8xx_gpu_get_slice_info(struct msm_gpu *gpu) 86 { 87 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 88 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 89 const struct a6xx_info *info = adreno_gpu->info->a6xx; 90 struct device *dev = &gpu->pdev->dev; 91 u32 slice_mask; 92 93 if (adreno_gpu->info->family < ADRENO_8XX_GEN1) 94 return; 95 96 if (a6xx_gpu->slice_mask) 97 return; 98 99 slice_mask = GENMASK(info->max_slices - 1, 0); 100 101 /* GEN1 doesn't support partial slice configurations */ 102 if (adreno_gpu->info->family == ADRENO_8XX_GEN1) { 103 a6xx_gpu->slice_mask = slice_mask; 104 return; 105 } 106 107 slice_mask &= a6xx_cx_misc_read(a6xx_gpu, 108 REG_A8XX_CX_MISC_SLICE_ENABLE_FINAL); 109 110 a6xx_gpu->slice_mask = slice_mask; 111 112 /* Chip ID depends on the number of slices available. So update it */ 113 adreno_gpu->chip_id |= FIELD_PREP(GENMASK(7, 4), hweight32(slice_mask)); 114 115 /* GMEM is split evenly between slices: report the active share */ 116 adreno_gpu->gmem = (adreno_gpu->info->gmem / info->max_slices) * hweight32(slice_mask); 117 118 /* Update the gpu-name to reflect the slice config: */ 119 const char *name = devm_kasprintf(dev, GFP_KERNEL, 120 "%"ADRENO_CHIPID_FMT, 121 ADRENO_CHIPID_ARGS(adreno_gpu->chip_id)); 122 if (name) { 123 devm_kfree(dev, adreno_gpu->base.name); 124 adreno_gpu->base.name = name; 125 } 126 } 127 128 static u32 a8xx_get_first_slice(struct a6xx_gpu *a6xx_gpu) 129 { 130 return ffs(a6xx_gpu->slice_mask) - 1; 131 } 132 133 static inline bool _a8xx_check_idle(struct msm_gpu *gpu) 134 { 135 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 136 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 137 138 /* Check that the GMU is idle */ 139 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu)) 140 return false; 141 142 /* Check that the CX master is idle */ 143 if (gpu_read(gpu, REG_A8XX_RBBM_STATUS) & 144 ~A8XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER) 145 return false; 146 147 return !(gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS) & 148 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT); 149 } 150 151 static bool a8xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 152 { 153 /* wait for CP to drain ringbuffer: */ 154 if (!adreno_idle(gpu, ring)) 155 return false; 156 157 if (spin_until(_a8xx_check_idle(gpu))) { 158 DRM_ERROR( 159 "%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", 160 gpu->name, __builtin_return_address(0), 161 gpu_read(gpu, REG_A8XX_RBBM_STATUS), 162 gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS), 163 gpu_read(gpu, REG_A6XX_CP_RB_RPTR), 164 gpu_read(gpu, REG_A6XX_CP_RB_WPTR)); 165 return false; 166 } 167 168 return true; 169 } 170 171 void a8xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 172 { 173 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 174 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 175 uint32_t wptr; 176 unsigned long flags; 177 178 spin_lock_irqsave(&ring->preempt_lock, flags); 179 180 /* Copy the shadow to the actual register */ 181 ring->cur = ring->next; 182 183 /* Make sure to wrap wptr if we need to */ 184 wptr = get_wptr(ring); 185 186 /* Update HW if this is the current ring and we are not in preempt*/ 187 if (!a6xx_in_preempt(a6xx_gpu)) { 188 if (a6xx_gpu->cur_ring == ring) 189 a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_RB_WPTR, wptr, BIT(0), false); 190 else 191 ring->restore_wptr = true; 192 } else { 193 ring->restore_wptr = true; 194 } 195 196 spin_unlock_irqrestore(&ring->preempt_lock, flags); 197 } 198 199 static void a8xx_set_hwcg(struct msm_gpu *gpu, bool state) 200 { 201 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 202 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 203 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 204 u32 val; 205 206 if ((adreno_is_x285_family(adreno_gpu) || adreno_is_a830(adreno_gpu)) && state) 207 gpu_write(gpu, REG_A8XX_RBBM_CGC_0_PC, 0x00000702); 208 209 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 210 state ? adreno_gpu->info->a6xx->gmu_cgc_mode : 0); 211 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 212 state ? 0x110111 : 0); 213 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 214 state ? 0x55555 : 0); 215 216 gpu_write(gpu, REG_A8XX_RBBM_CLOCK_CNTL_GLOBAL, 1); 217 gpu_write(gpu, REG_A8XX_RBBM_CGC_GLOBAL_LOAD_CMD, !!state); 218 219 if (state) { 220 gpu_write(gpu, REG_A8XX_RBBM_CGC_P2S_TRIG_CMD, 1); 221 222 if (gpu_poll_timeout(gpu, REG_A8XX_RBBM_CGC_P2S_STATUS, val, 223 val & A8XX_RBBM_CGC_P2S_STATUS_TXDONE, 1, 10)) { 224 dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n"); 225 return; 226 } 227 228 gpu_write(gpu, REG_A8XX_RBBM_CLOCK_CNTL_GLOBAL, 0); 229 } else { 230 /* 231 * GMU enables clk gating in GBIF during boot up. So, 232 * override that here when hwcg feature is disabled 233 */ 234 gpu_rmw(gpu, REG_A6XX_GBIF_CX_CONFIG, BIT(0), 0); 235 } 236 } 237 238 static void a8xx_set_cp_protect(struct msm_gpu *gpu) 239 { 240 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 241 const struct adreno_protect *protect = adreno_gpu->info->a6xx->protect; 242 u32 cntl, final_cfg; 243 unsigned int i; 244 245 cntl = A8XX_CP_PROTECT_CNTL_PIPE_ACCESS_PROT_EN | 246 A8XX_CP_PROTECT_CNTL_PIPE_ACCESS_FAULT_ON_VIOL_EN | 247 A8XX_CP_PROTECT_CNTL_PIPE_LAST_SPAN_INF_RANGE | 248 A8XX_CP_PROTECT_CNTL_PIPE_HALT_SQE_RANGE__MASK; 249 /* 250 * Enable access protection to privileged registers, fault on an access 251 * protect violation and select the last span to protect from the start 252 * address all the way to the end of the register address space 253 */ 254 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_CP_PROTECT_CNTL_PIPE, cntl); 255 a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_CP_PROTECT_CNTL_PIPE, cntl); 256 257 a8xx_aperture_clear(gpu); 258 259 for (i = 0; i < protect->count; i++) { 260 /* Intentionally skip writing to some registers */ 261 if (protect->regs[i]) { 262 gpu_write(gpu, REG_A8XX_CP_PROTECT_GLOBAL(i), protect->regs[i]); 263 final_cfg = protect->regs[i]; 264 } 265 } 266 267 /* 268 * Last span feature is only supported on PIPE specific register. 269 * So update those here 270 */ 271 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_CP_PROTECT_PIPE(15), final_cfg); 272 a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_CP_PROTECT_PIPE(15), final_cfg); 273 274 a8xx_aperture_clear(gpu); 275 } 276 277 static void a8xx_set_ubwc_config(struct msm_gpu *gpu) 278 { 279 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 280 const struct qcom_ubwc_cfg_data *cfg = adreno_gpu->ubwc_config; 281 u32 level2_swizzling_dis = !(qcom_ubwc_swizzle(cfg) & UBWC_SWIZZLE_ENABLE_LVL2); 282 u32 level3_swizzling_dis = !(qcom_ubwc_swizzle(cfg) & UBWC_SWIZZLE_ENABLE_LVL3); 283 bool rgba8888_lossless = false, fp16compoptdis = false; 284 bool yuvnotcomptofc = false, min_acc_len_64b = false; 285 bool rgb565_predicator = false; 286 bool amsbc = qcom_ubwc_enable_amsbc(cfg); 287 bool ubwc_mode = qcom_ubwc_get_ubwc_mode(cfg); 288 u32 ubwc_version = cfg->ubwc_enc_version; 289 u32 hbb, hbb_hi, hbb_lo, mode; 290 u8 uavflagprd_inv = 2; 291 292 if (ubwc_version > UBWC_7_0) 293 dev_err(&gpu->pdev->dev, "Unknown UBWC version: 0x%x\n", ubwc_version); 294 295 if (ubwc_version == UBWC_6_0 || 296 ubwc_version == UBWC_7_0) 297 yuvnotcomptofc = true; 298 299 if (ubwc_version < UBWC_5_0 && 300 ubwc_version >= UBWC_4_0) 301 rgba8888_lossless = true; 302 303 if (ubwc_version < UBWC_4_3) 304 fp16compoptdis = true; 305 306 if (cfg->ubwc_enc_version >= UBWC_4_0) 307 rgb565_predicator = true; 308 309 if (ubwc_version < UBWC_3_0) 310 dev_err(&gpu->pdev->dev, "Unsupported UBWC version: 0x%x\n", ubwc_version); 311 312 mode = qcom_ubwc_version_tag(cfg); 313 314 /* 315 * We subtract 13 from the highest bank bit (13 is the minimum value 316 * allowed by hw) and write the lowest two bits of the remaining value 317 * as hbb_lo and the one above it as hbb_hi to the hardware. 318 */ 319 WARN_ON(cfg->highest_bank_bit < 13); 320 hbb = cfg->highest_bank_bit - 13; 321 hbb_hi = hbb >> 2; 322 hbb_lo = hbb & 3; 323 324 a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_GRAS_NC_MODE_CNTL, 325 hbb << 5 | 326 level3_swizzling_dis << 4 | 327 level2_swizzling_dis << 3); 328 329 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_GRAS_NC_MODE_CNTL, 330 hbb << 5 | 331 level3_swizzling_dis << 4 | 332 level2_swizzling_dis << 3); 333 334 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_CCU_NC_MODE_CNTL, 335 yuvnotcomptofc << 6 | 336 level3_swizzling_dis << 5 | 337 level2_swizzling_dis << 4 | 338 hbb_hi << 3 | 339 hbb_lo << 1); 340 341 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_CMP_NC_MODE_CNTL, 342 mode << 15 | 343 yuvnotcomptofc << 6 | 344 rgba8888_lossless << 4 | 345 fp16compoptdis << 3 | 346 rgb565_predicator << 2 | 347 amsbc << 1 | 348 min_acc_len_64b); 349 350 a8xx_aperture_clear(gpu); 351 352 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 353 level3_swizzling_dis << 13 | 354 level2_swizzling_dis << 12 | 355 hbb_hi << 10 | 356 uavflagprd_inv << 4 | 357 min_acc_len_64b << 3 | 358 hbb_lo << 1 | ubwc_mode); 359 360 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 361 level3_swizzling_dis << 7 | 362 level2_swizzling_dis << 6 | 363 hbb_hi << 4 | 364 min_acc_len_64b << 3 | 365 hbb_lo << 1 | ubwc_mode); 366 } 367 368 static void a8xx_nonctxt_config(struct msm_gpu *gpu, u32 *gmem_protect) 369 { 370 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 371 const struct a6xx_info *info = adreno_gpu->info->a6xx; 372 const struct adreno_reglist_pipe *regs = info->nonctxt_reglist; 373 unsigned int pipe_id, i; 374 unsigned long flags; 375 376 for (pipe_id = PIPE_NONE; pipe_id <= PIPE_DDE_BV; pipe_id++) { 377 /* We don't have support for LPAC yet */ 378 if (pipe_id == PIPE_LPAC) 379 continue; 380 381 a8xx_aperture_acquire(gpu, pipe_id, &flags); 382 383 for (i = 0; regs[i].offset; i++) { 384 if (!(BIT(pipe_id) & regs[i].pipe)) 385 continue; 386 387 if (regs[i].offset == REG_A8XX_RB_GC_GMEM_PROTECT) 388 *gmem_protect = regs[i].value; 389 390 gpu_write(gpu, regs[i].offset, regs[i].value); 391 } 392 393 a8xx_aperture_release(gpu, flags); 394 } 395 396 a8xx_aperture_clear(gpu); 397 } 398 399 static void a8xx_patch_pwrup_reglist(struct msm_gpu *gpu) 400 { 401 const struct adreno_reglist_pipe_list *dyn_pwrup_reglist; 402 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 403 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 404 const struct adreno_reglist_list *reglist; 405 void *ptr = a6xx_gpu->pwrup_reglist_ptr; 406 struct cpu_gpu_lock *lock = ptr; 407 u32 *dest = (u32 *)&lock->regs[0]; 408 u32 dyn_pwrup_reglist_count = 0; 409 int i; 410 411 lock->gpu_req = lock->cpu_req = lock->turn = 0; 412 413 reglist = adreno_gpu->info->a6xx->ifpc_reglist; 414 if (reglist) { 415 lock->ifpc_list_len = reglist->count; 416 417 /* 418 * For each entry in each of the lists, write the offset and the current 419 * register value into the GPU buffer 420 */ 421 for (i = 0; i < reglist->count; i++) { 422 *dest++ = reglist->regs[i]; 423 *dest++ = gpu_read(gpu, reglist->regs[i]); 424 } 425 } 426 427 reglist = adreno_gpu->info->a6xx->pwrup_reglist; 428 if (reglist) { 429 lock->preemption_list_len = reglist->count; 430 431 for (i = 0; i < reglist->count; i++) { 432 *dest++ = reglist->regs[i]; 433 *dest++ = gpu_read(gpu, reglist->regs[i]); 434 } 435 } 436 437 /* 438 * The overall register list is composed of 439 * 1. Static IFPC-only registers 440 * 2. Static IFPC + preemption registers 441 * 3. Dynamic IFPC + preemption registers (ex: perfcounter selects) 442 * 443 * The first two lists are static. Size of these lists are stored as 444 * number of pairs in ifpc_list_len and preemption_list_len 445 * respectively. With concurrent binning, Some of the perfcounter 446 * registers being virtualized, CP needs to know the pipe id to program 447 * the aperture inorder to restore the same. Thus, third list is a 448 * dynamic list with triplets as 449 * (<aperture, shifted 12 bits> <address> <data>), and the length is 450 * stored as number for triplets in dynamic_list_len. 451 */ 452 dyn_pwrup_reglist = adreno_gpu->info->a6xx->dyn_pwrup_reglist; 453 if (!dyn_pwrup_reglist) 454 goto done; 455 456 for (u32 pipe_id = PIPE_BR; pipe_id <= PIPE_DDE_BV; pipe_id++) { 457 for (i = 0; i < dyn_pwrup_reglist->count; i++) { 458 if (!(dyn_pwrup_reglist->regs[i].pipe & BIT(pipe_id))) 459 continue; 460 *dest++ = A8XX_CP_APERTURE_CNTL_HOST_PIPEID(pipe_id); 461 *dest++ = dyn_pwrup_reglist->regs[i].offset; 462 *dest++ = a8xx_read_pipe_slice(gpu, 463 pipe_id, 464 a8xx_get_first_slice(a6xx_gpu), 465 dyn_pwrup_reglist->regs[i].offset); 466 dyn_pwrup_reglist_count++; 467 } 468 } 469 470 lock->dynamic_list_len = dyn_pwrup_reglist_count; 471 a6xx_gpu->dynamic_sel_reglist_offset = dyn_pwrup_reglist_count; 472 473 done: 474 a8xx_aperture_clear(gpu); 475 } 476 477 static int a8xx_preempt_start(struct msm_gpu *gpu) 478 { 479 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 480 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 481 struct msm_ringbuffer *ring = gpu->rb[0]; 482 483 if (gpu->nr_rings <= 1) 484 return 0; 485 486 /* Turn CP protection off */ 487 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 488 OUT_RING(ring, 0); 489 490 a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, NULL); 491 492 a6xx_flush_yield(gpu, ring); 493 494 return a8xx_idle(gpu, ring) ? 0 : -EINVAL; 495 } 496 497 static int a8xx_cp_init(struct msm_gpu *gpu) 498 { 499 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 500 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 501 struct msm_ringbuffer *ring = gpu->rb[0]; 502 u32 mask; 503 504 /* Disable concurrent binning before sending CP init */ 505 OUT_PKT7(ring, CP_THREAD_CONTROL, 1); 506 OUT_RING(ring, BIT(27)); 507 508 OUT_PKT7(ring, CP_ME_INIT, 7); 509 510 /* Use multiple HW contexts */ 511 mask = BIT(0); 512 513 /* Enable error detection */ 514 mask |= BIT(1); 515 516 /* Set default reset state */ 517 mask |= BIT(3); 518 519 /* Disable save/restore of performance counters across preemption */ 520 mask |= BIT(6); 521 522 /* Enable the register init list with the spinlock */ 523 mask |= BIT(8); 524 525 OUT_RING(ring, mask); 526 527 /* Enable multiple hardware contexts */ 528 OUT_RING(ring, 0x00000003); 529 530 /* Enable error detection */ 531 OUT_RING(ring, 0x20000000); 532 533 /* Operation mode mask */ 534 OUT_RING(ring, 0x00000002); 535 536 /* Lo address */ 537 OUT_RING(ring, lower_32_bits(a6xx_gpu->pwrup_reglist_iova)); 538 /* Hi address */ 539 OUT_RING(ring, upper_32_bits(a6xx_gpu->pwrup_reglist_iova)); 540 541 /* Enable dyn pwrup list with triplets (offset, value, pipe) */ 542 OUT_RING(ring, BIT(31)); 543 544 a6xx_flush(gpu, ring); 545 return a8xx_idle(gpu, ring) ? 0 : -EINVAL; 546 } 547 548 #define A8XX_INT_MASK \ 549 (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \ 550 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \ 551 A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \ 552 A6XX_RBBM_INT_0_MASK_CP_SW | \ 553 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ 554 A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \ 555 A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \ 556 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ 557 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \ 558 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \ 559 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ 560 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \ 561 A6XX_RBBM_INT_0_MASK_TSBWRITEERROR | \ 562 A6XX_RBBM_INT_0_MASK_SWFUSEVIOLATION) 563 564 #define A8XX_APRIV_MASK \ 565 (A8XX_CP_APRIV_CNTL_PIPE_ICACHE | \ 566 A8XX_CP_APRIV_CNTL_PIPE_RBFETCH | \ 567 A8XX_CP_APRIV_CNTL_PIPE_RBPRIVLEVEL | \ 568 A8XX_CP_APRIV_CNTL_PIPE_RBRPWB) 569 570 #define A8XX_BR_APRIV_MASK \ 571 (A8XX_APRIV_MASK | \ 572 A8XX_CP_APRIV_CNTL_PIPE_CDREAD | \ 573 A8XX_CP_APRIV_CNTL_PIPE_CDWRITE) 574 575 #define A8XX_CP_GLOBAL_INT_MASK \ 576 (A8XX_CP_GLOBAL_INT_MASK_HWFAULTBR | \ 577 A8XX_CP_GLOBAL_INT_MASK_HWFAULTBV | \ 578 A8XX_CP_GLOBAL_INT_MASK_HWFAULTLPAC | \ 579 A8XX_CP_GLOBAL_INT_MASK_HWFAULTAQE0 | \ 580 A8XX_CP_GLOBAL_INT_MASK_HWFAULTAQE1 | \ 581 A8XX_CP_GLOBAL_INT_MASK_HWFAULTDDEBR | \ 582 A8XX_CP_GLOBAL_INT_MASK_HWFAULTDDEBV | \ 583 A8XX_CP_GLOBAL_INT_MASK_SWFAULTBR | \ 584 A8XX_CP_GLOBAL_INT_MASK_SWFAULTBV | \ 585 A8XX_CP_GLOBAL_INT_MASK_SWFAULTLPAC | \ 586 A8XX_CP_GLOBAL_INT_MASK_SWFAULTAQE0 | \ 587 A8XX_CP_GLOBAL_INT_MASK_SWFAULTAQE1 | \ 588 A8XX_CP_GLOBAL_INT_MASK_SWFAULTDDEBR | \ 589 A8XX_CP_GLOBAL_INT_MASK_SWFAULTDDEBV) 590 591 #define A8XX_CP_INTERRUPT_STATUS_MASK_PIPE \ 592 (A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFRBWRAP | \ 593 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB1WRAP | \ 594 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB2WRAP | \ 595 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB3WRAP | \ 596 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFSDSWRAP | \ 597 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFMRBWRAP | \ 598 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFVSDWRAP | \ 599 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_OPCODEERROR | \ 600 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VSDPARITYERROR | \ 601 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_REGISTERPROTECTIONERROR | \ 602 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_ILLEGALINSTRUCTION | \ 603 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_SMMUFAULT | \ 604 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPCLIENT| \ 605 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPTYPE | \ 606 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPREAD | \ 607 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESP | \ 608 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_RTWROVF | \ 609 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTWROVF | \ 610 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTREFCNTOVF | \ 611 A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTCLRRESMISS) 612 613 #define A8XX_CP_HW_FAULT_STATUS_MASK_PIPE \ 614 (A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFRBFAULT | \ 615 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB1FAULT | \ 616 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB2FAULT | \ 617 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB3FAULT | \ 618 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFSDSFAULT | \ 619 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFMRBFAULT | \ 620 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFVSDFAULT | \ 621 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_SQEREADBURSTOVF | \ 622 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_EVENTENGINEOVF | \ 623 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_UCODEERROR) 624 625 static int hw_init(struct msm_gpu *gpu) 626 { 627 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 628 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 629 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 630 unsigned int pipe_id, i; 631 u32 gmem_protect = 0; 632 u64 gmem_range_min; 633 int ret; 634 635 ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 636 if (ret) 637 return ret; 638 639 /* Clear the cached value to force aperture configuration next time */ 640 a6xx_gpu->cached_aperture = UINT_MAX; 641 a8xx_aperture_clear(gpu); 642 643 /* Clear GBIF halt in case GX domain was not collapsed */ 644 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0); 645 gpu_read(gpu, REG_A6XX_GBIF_HALT); 646 647 gpu_write(gpu, REG_A8XX_RBBM_GBIF_HALT, 0); 648 gpu_read(gpu, REG_A8XX_RBBM_GBIF_HALT); 649 650 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); 651 652 /* 653 * Disable the trusted memory range - we don't actually supported secure 654 * memory rendering at this point in time and we don't want to block off 655 * part of the virtual memory space. 656 */ 657 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000); 658 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); 659 660 /* Make all blocks contribute to the GPU BUSY perf counter */ 661 gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff); 662 663 /* Setup GMEM Range in UCHE */ 664 gmem_range_min = SZ_64M; 665 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ 666 gpu_write64(gpu, REG_A8XX_UCHE_CCHE_GC_GMEM_RANGE_MIN, gmem_range_min); 667 gpu_write64(gpu, REG_A8XX_SP_HLSQ_GC_GMEM_RANGE_MIN, gmem_range_min); 668 669 /* Setup UCHE Trap region */ 670 gpu_write64(gpu, REG_A8XX_UCHE_TRAP_BASE, adreno_gpu->uche_trap_base); 671 gpu_write64(gpu, REG_A8XX_UCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base); 672 gpu_write64(gpu, REG_A8XX_UCHE_CCHE_TRAP_BASE, adreno_gpu->uche_trap_base); 673 gpu_write64(gpu, REG_A8XX_UCHE_CCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base); 674 675 /* Turn on performance counters */ 676 gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_CNTL, 0x1); 677 gpu_write(gpu, REG_A8XX_RBBM_SLICE_PERFCTR_CNTL, 0x1); 678 679 /* Turn on the IFPC counter (countable 4 on XOCLK1) */ 680 gmu_write(&a6xx_gpu->gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_SELECT_XOCLK_1, 681 FIELD_PREP(GENMASK(7, 0), 0x4)); 682 683 /* Select CP0 to always count cycles */ 684 gpu_write(gpu, REG_A8XX_CP_PERFCTR_CP_SEL(0), 1); 685 686 a8xx_set_ubwc_config(gpu); 687 688 /* Set weights for bicubic filtering */ 689 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(0), 0); 690 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(1), 0x3fe05ff4); 691 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(2), 0x3fa0ebee); 692 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(3), 0x3f5193ed); 693 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(4), 0x3f0243f0); 694 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(5), 0x00000000); 695 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(6), 0x3fd093e8); 696 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(7), 0x3f4133dc); 697 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(8), 0x3ea1dfdb); 698 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(9), 0x3e0283e0); 699 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(10), 0x0000ac2b); 700 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(11), 0x0000f01d); 701 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(12), 0x00114412); 702 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(13), 0x0021980a); 703 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(14), 0x0051ec05); 704 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(15), 0x0000380e); 705 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(16), 0x3ff09001); 706 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(17), 0x3fc10bfa); 707 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(18), 0x3f9193f7); 708 gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(19), 0x3f7227f7); 709 710 gpu_write(gpu, REG_A8XX_UCHE_CLIENT_PF, BIT(7) | 0x1); 711 712 a8xx_nonctxt_config(gpu, &gmem_protect); 713 714 /* Enable fault detection */ 715 gpu_write(gpu, REG_A8XX_RBBM_INTERFACE_HANG_INT_CNTL, BIT(30) | 0xcfffff); 716 gpu_write(gpu, REG_A8XX_RBBM_SLICE_INTERFACE_HANG_INT_CNTL, BIT(30)); 717 718 /* Set up the CX GMU counter 0 to count busy ticks */ 719 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000); 720 721 /* Enable the power counter */ 722 gmu_rmw(gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_SELECT_XOCLK_0, 0xff, BIT(5)); 723 gmu_write(gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1); 724 725 /* Protect registers from the CP */ 726 a8xx_set_cp_protect(gpu); 727 728 /* Enable the GMEM save/restore feature for preemption */ 729 a8xx_write_pipe(gpu, PIPE_BR, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE, 1); 730 731 for (pipe_id = PIPE_BR; pipe_id <= PIPE_DDE_BV; pipe_id++) { 732 u32 apriv_mask = A8XX_APRIV_MASK; 733 u32 int_mask = A8XX_CP_INTERRUPT_STATUS_MASK_PIPE; 734 unsigned long flags; 735 736 if (pipe_id == PIPE_LPAC) 737 continue; 738 739 if (pipe_id == PIPE_BR) 740 apriv_mask = A8XX_BR_APRIV_MASK; 741 742 /* 743 * A845/A850's BV pipe raises a spurious RTWROVF fault interrupt; 744 * mask it off to avoid false fault recovery. 745 */ 746 if (adreno_is_a850_family(adreno_gpu) && pipe_id == PIPE_BV) 747 int_mask &= ~A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_RTWROVF; 748 749 a8xx_aperture_acquire(gpu, pipe_id, &flags); 750 gpu_write(gpu, REG_A8XX_CP_APRIV_CNTL_PIPE, apriv_mask); 751 gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_PIPE, int_mask); 752 gpu_write(gpu, REG_A8XX_CP_HW_FAULT_STATUS_MASK_PIPE, 753 A8XX_CP_HW_FAULT_STATUS_MASK_PIPE); 754 a8xx_aperture_release(gpu, flags); 755 } 756 757 a8xx_aperture_clear(gpu); 758 759 /* Enable interrupts */ 760 gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_GLOBAL, A8XX_CP_GLOBAL_INT_MASK); 761 gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A8XX_INT_MASK); 762 763 ret = adreno_hw_init(gpu); 764 if (ret) 765 goto out; 766 767 gpu_write64(gpu, REG_A8XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova); 768 if (a6xx_gpu->aqe_iova) 769 gpu_write64(gpu, REG_A8XX_CP_AQE_INSTR_BASE_0, a6xx_gpu->aqe_iova); 770 771 /* Set the ringbuffer address */ 772 gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova); 773 gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT); 774 775 /* Configure the RPTR shadow if needed: */ 776 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR, shadowptr(a6xx_gpu, gpu->rb[0])); 777 gpu_write64(gpu, REG_A8XX_CP_RB_RPTR_ADDR_BV, rbmemptr(gpu->rb[0], bv_rptr)); 778 779 a8xx_preempt_hw_init(gpu); 780 781 for (i = 0; i < gpu->nr_rings; i++) 782 a6xx_gpu->shadow[i] = 0; 783 784 /* Always come up on rb 0 */ 785 a6xx_gpu->cur_ring = gpu->rb[0]; 786 787 for (i = 0; i < gpu->nr_rings; i++) 788 gpu->rb[i]->cur_ctx_seqno = 0; 789 790 /* Enable the SQE_to start the CP engine */ 791 gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 1); 792 793 ret = a8xx_cp_init(gpu); 794 if (ret) 795 goto out; 796 797 /* 798 * Try to load a zap shader into the secure world. If successful 799 * we can use the CP to switch out of secure mode. If not then we 800 * have no resource but to try to switch ourselves out manually. If we 801 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will 802 * be blocked and a permissions violation will soon follow. 803 */ 804 ret = a6xx_zap_shader_init(gpu); 805 if (!ret) { 806 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); 807 OUT_RING(gpu->rb[0], 0x00000000); 808 809 a6xx_flush(gpu, gpu->rb[0]); 810 if (!a8xx_idle(gpu, gpu->rb[0])) 811 return -EINVAL; 812 } else if (ret == -ENODEV) { 813 /* 814 * This device does not use zap shader (but print a warning 815 * just in case someone got their dt wrong.. hopefully they 816 * have a debug UART to realize the error of their ways... 817 * if you mess this up you are about to crash horribly) 818 */ 819 dev_warn_once(gpu->dev->dev, 820 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); 821 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0); 822 ret = 0; 823 } else { 824 return ret; 825 } 826 827 /* 828 * GMEM_PROTECT register should be programmed after GPU is transitioned to 829 * non-secure mode 830 */ 831 a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_GC_GMEM_PROTECT, gmem_protect); 832 WARN_ON(!gmem_protect); 833 a8xx_aperture_clear(gpu); 834 835 if (!a6xx_gpu->pwrup_reglist_emitted) { 836 a8xx_patch_pwrup_reglist(gpu); 837 a6xx_gpu->pwrup_reglist_emitted = true; 838 } 839 840 /* Enable hardware clockgating */ 841 a8xx_set_hwcg(gpu, true); 842 out: 843 /* Last step - yield the ringbuffer */ 844 a8xx_preempt_start(gpu); 845 846 /* 847 * Tell the GMU that we are done touching the GPU and it can start power 848 * management 849 */ 850 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); 851 852 if (!ret && msm_gpu_sysprof_no_perfcntr_zap(gpu)) { 853 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); 854 if (!ret) 855 set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status); 856 } 857 858 return ret; 859 } 860 861 int a8xx_hw_init(struct msm_gpu *gpu) 862 { 863 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 864 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 865 int ret; 866 867 mutex_lock(&a6xx_gpu->gmu.lock); 868 ret = hw_init(gpu); 869 mutex_unlock(&a6xx_gpu->gmu.lock); 870 871 return ret; 872 } 873 874 static void a8xx_dump(struct msm_gpu *gpu) 875 { 876 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n", gpu_read(gpu, REG_A8XX_RBBM_STATUS)); 877 adreno_dump(gpu); 878 } 879 880 void a8xx_recover(struct msm_gpu *gpu) 881 { 882 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 883 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 884 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; 885 int active_submits; 886 887 adreno_dump_info(gpu); 888 889 /* 890 * To handle recovery specific sequences during the rpm suspend we are 891 * about to trigger 892 */ 893 a6xx_gpu->hung = true; 894 895 if (adreno_gpu->funcs->gx_is_on(adreno_gpu)) { 896 /* 897 * Sometimes crashstate capture is skipped, so SQE should be 898 * halted here again 899 */ 900 gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 3); 901 902 if (hang_debug) 903 a8xx_dump(gpu); 904 } 905 906 pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); 907 908 /* active_submit won't change until we make a submission */ 909 mutex_lock(&gpu->active_lock); 910 active_submits = gpu->active_submits; 911 912 /* 913 * Temporarily clear active_submits count to silence a WARN() in the 914 * runtime suspend cb 915 */ 916 gpu->active_submits = 0; 917 918 reinit_completion(&gmu->pd_gate); 919 dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb); 920 dev_pm_genpd_synced_poweroff(gmu->cxpd); 921 922 /* Drop the rpm refcount from active submits */ 923 if (active_submits) 924 pm_runtime_put(&gpu->pdev->dev); 925 926 /* And the final one from recover worker */ 927 pm_runtime_put_sync(&gpu->pdev->dev); 928 929 if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000))) 930 DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n"); 931 932 dev_pm_genpd_remove_notifier(gmu->cxpd); 933 934 pm_runtime_use_autosuspend(&gpu->pdev->dev); 935 936 if (active_submits) 937 pm_runtime_get(&gpu->pdev->dev); 938 939 pm_runtime_get_sync(&gpu->pdev->dev); 940 941 gpu->active_submits = active_submits; 942 mutex_unlock(&gpu->active_lock); 943 944 msm_gpu_hw_init(gpu); 945 a6xx_gpu->hung = false; 946 } 947 948 static const char *a8xx_uche_fault_block(struct msm_gpu *gpu, u32 mid) 949 { 950 static const char * const uche_clients[] = { 951 "BR_VFD", "BR_SP", "BR_VSC", "BR_VPC", "BR_HLSQ", "BR_PC", "BR_LRZ", "BR_TP", 952 "BV_VFD", "BV_SP", "BV_VSC", "BV_VPC", "BV_HLSQ", "BV_PC", "BV_LRZ", "BV_TP", 953 "STCHE", 954 }; 955 static const char * const uche_clients_lpac[] = { 956 "-", "SP_LPAC", "-", "-", "HLSQ_LPAC", "-", "-", "TP_LPAC", 957 }; 958 u32 val; 959 960 /* 961 * The source of the data depends on the mid ID read from FSYNR1. 962 * and the client ID read from the UCHE block 963 */ 964 val = gpu_read(gpu, REG_A8XX_UCHE_CLIENT_PF); 965 966 val &= GENMASK(6, 0); 967 968 /* mid=3 refers to BR or BV */ 969 if (mid == 3) { 970 if (val < ARRAY_SIZE(uche_clients)) 971 return uche_clients[val]; 972 else 973 return "UCHE"; 974 } 975 976 /* mid=8 refers to LPAC */ 977 if (mid == 8) { 978 if (val < ARRAY_SIZE(uche_clients_lpac)) 979 return uche_clients_lpac[val]; 980 else 981 return "UCHE_LPAC"; 982 } 983 984 return "Unknown"; 985 } 986 987 static const char *a8xx_fault_block(struct msm_gpu *gpu, u32 id) 988 { 989 switch (id) { 990 case 0x0: 991 return "CP"; 992 case 0x1: 993 return "UCHE: Unknown"; 994 case 0x2: 995 return "UCHE_LPAC: Unknown"; 996 case 0x3: 997 case 0x8: 998 return a8xx_uche_fault_block(gpu, id); 999 case 0x4: 1000 return "CCU"; 1001 case 0x5: 1002 return "Flag cache"; 1003 case 0x6: 1004 return "PREFETCH"; 1005 case 0x7: 1006 return "GMU"; 1007 case 0x9: 1008 return "UCHE_HPAC"; 1009 } 1010 1011 return "Unknown"; 1012 } 1013 1014 int a8xx_fault_handler(void *arg, unsigned long iova, int flags, void *data) 1015 { 1016 struct msm_gpu *gpu = arg; 1017 struct adreno_smmu_fault_info *info = data; 1018 const char *block = "unknown"; 1019 1020 u32 scratch[] = { 1021 gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(0)), 1022 gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(1)), 1023 gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(2)), 1024 gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(3)), 1025 }; 1026 1027 if (info) 1028 block = a8xx_fault_block(gpu, info->fsynr1 & 0xff); 1029 1030 return adreno_fault_handler(gpu, iova, flags, info, block, scratch); 1031 } 1032 1033 static void a8xx_cp_hw_err_irq(struct msm_gpu *gpu) 1034 { 1035 u32 status = gpu_read(gpu, REG_A8XX_CP_INTERRUPT_STATUS_GLOBAL); 1036 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1037 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 1038 u32 slice = a8xx_get_first_slice(a6xx_gpu); 1039 u32 hw_fault_mask = GENMASK(6, 0); 1040 u32 sw_fault_mask = GENMASK(22, 16); 1041 u32 pipe = 0; 1042 1043 dev_err_ratelimited(&gpu->pdev->dev, "CP Fault Global INT status: 0x%x\n", status); 1044 1045 if (status & (A8XX_CP_GLOBAL_INT_MASK_HWFAULTBR | 1046 A8XX_CP_GLOBAL_INT_MASK_SWFAULTBR)) 1047 pipe |= BIT(PIPE_BR); 1048 1049 if (status & (A8XX_CP_GLOBAL_INT_MASK_HWFAULTBV | 1050 A8XX_CP_GLOBAL_INT_MASK_SWFAULTBV)) 1051 pipe |= BIT(PIPE_BV); 1052 1053 if (!pipe) { 1054 dev_err_ratelimited(&gpu->pdev->dev, "CP Fault Unknown pipe\n"); 1055 goto out; 1056 } 1057 1058 for (unsigned int pipe_id = PIPE_NONE; pipe_id <= PIPE_DDE_BV; pipe_id++) { 1059 if (!(BIT(pipe_id) & pipe)) 1060 continue; 1061 1062 if (hw_fault_mask & status) { 1063 status = a8xx_read_pipe_slice(gpu, pipe_id, slice, 1064 REG_A8XX_CP_HW_FAULT_STATUS_PIPE); 1065 dev_err_ratelimited(&gpu->pdev->dev, 1066 "CP HW FAULT pipe: %u status: 0x%x\n", pipe_id, status); 1067 } 1068 1069 if (sw_fault_mask & status) { 1070 status = a8xx_read_pipe_slice(gpu, pipe_id, slice, 1071 REG_A8XX_CP_INTERRUPT_STATUS_PIPE); 1072 dev_err_ratelimited(&gpu->pdev->dev, 1073 "CP SW FAULT pipe: %u status: 0x%x\n", pipe_id, status); 1074 1075 if (status & BIT(8)) { 1076 a8xx_write_pipe(gpu, pipe_id, REG_A8XX_CP_SQE_STAT_ADDR_PIPE, 1); 1077 status = a8xx_read_pipe_slice(gpu, pipe_id, slice, 1078 REG_A8XX_CP_SQE_STAT_DATA_PIPE); 1079 dev_err_ratelimited(&gpu->pdev->dev, 1080 "CP Opcode error, opcode=0x%x\n", status); 1081 } 1082 1083 if (status & BIT(10)) { 1084 status = a8xx_read_pipe_slice(gpu, pipe_id, slice, 1085 REG_A8XX_CP_PROTECT_STATUS_PIPE); 1086 dev_err_ratelimited(&gpu->pdev->dev, 1087 "CP REG PROTECT error, status=0x%x\n", status); 1088 } 1089 } 1090 } 1091 1092 out: 1093 /* Turn off interrupts to avoid triggering recovery again */ 1094 a8xx_aperture_clear(gpu); 1095 gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_GLOBAL, 0); 1096 gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, 0); 1097 1098 kthread_queue_work(gpu->worker, &gpu->recover_work); 1099 } 1100 1101 static u32 gpu_periph_read(struct msm_gpu *gpu, u32 dbg_offset) 1102 { 1103 gpu_write(gpu, REG_A8XX_CP_SQE_UCODE_DBG_ADDR_PIPE, dbg_offset); 1104 1105 return gpu_read(gpu, REG_A8XX_CP_SQE_UCODE_DBG_DATA_PIPE); 1106 } 1107 1108 static u64 gpu_periph_read64(struct msm_gpu *gpu, u32 dbg_offset) 1109 { 1110 u64 lo, hi; 1111 1112 lo = gpu_periph_read(gpu, dbg_offset); 1113 hi = gpu_periph_read(gpu, dbg_offset + 1); 1114 1115 return (hi << 32) | lo; 1116 } 1117 1118 #define CP_PERIPH_IB1_BASE_LO 0x7005 1119 #define CP_PERIPH_IB1_BASE_HI 0x7006 1120 #define CP_PERIPH_IB1_SIZE 0x7007 1121 #define CP_PERIPH_IB1_OFFSET 0x7008 1122 #define CP_PERIPH_IB2_BASE_LO 0x7009 1123 #define CP_PERIPH_IB2_BASE_HI 0x700a 1124 #define CP_PERIPH_IB2_SIZE 0x700b 1125 #define CP_PERIPH_IB2_OFFSET 0x700c 1126 #define CP_PERIPH_IB3_BASE_LO 0x700d 1127 #define CP_PERIPH_IB3_BASE_HI 0x700e 1128 #define CP_PERIPH_IB3_SIZE 0x700f 1129 #define CP_PERIPH_IB3_OFFSET 0x7010 1130 1131 static void a8xx_fault_detect_irq(struct msm_gpu *gpu) 1132 { 1133 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1134 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 1135 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); 1136 unsigned long flags; 1137 1138 /* 1139 * If stalled on SMMU fault, we could trip the GPU's hang detection, 1140 * but the fault handler will trigger the devcore dump, and we want 1141 * to otherwise resume normally rather than killing the submit, so 1142 * just bail. 1143 */ 1144 if (gpu_read(gpu, REG_A8XX_RBBM_MISC_STATUS) & A8XX_RBBM_MISC_STATUS_SMMU_STALLED_ON_FAULT) 1145 return; 1146 1147 /* 1148 * Force the GPU to stay on until after we finish 1149 * collecting information 1150 */ 1151 if (!adreno_has_gmu_wrapper(adreno_gpu)) 1152 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); 1153 1154 DRM_DEV_ERROR(&gpu->pdev->dev, 1155 "gpu fault ring %d fence %x status %8.8X gfx_status %8.8X\n", 1156 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, 1157 gpu_read(gpu, REG_A8XX_RBBM_STATUS), gpu_read(gpu, REG_A8XX_RBBM_GFX_STATUS)); 1158 1159 a8xx_aperture_acquire(gpu, PIPE_BR, &flags); 1160 1161 DRM_DEV_ERROR(&gpu->pdev->dev, 1162 "BR: status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x ib3 %16.16llX/%4.4x\n", 1163 gpu_read(gpu, REG_A8XX_RBBM_GFX_BR_STATUS), 1164 gpu_read(gpu, REG_A6XX_CP_RB_RPTR), 1165 gpu_read(gpu, REG_A6XX_CP_RB_WPTR), 1166 gpu_periph_read64(gpu, CP_PERIPH_IB1_BASE_LO), 1167 gpu_periph_read(gpu, CP_PERIPH_IB1_OFFSET), 1168 gpu_periph_read64(gpu, CP_PERIPH_IB2_BASE_LO), 1169 gpu_periph_read(gpu, CP_PERIPH_IB2_OFFSET), 1170 gpu_periph_read64(gpu, CP_PERIPH_IB3_BASE_LO), 1171 gpu_periph_read(gpu, CP_PERIPH_IB3_OFFSET)); 1172 1173 a8xx_aperture_release(gpu, flags); 1174 a8xx_aperture_acquire(gpu, PIPE_BV, &flags); 1175 1176 DRM_DEV_ERROR(&gpu->pdev->dev, 1177 "BV: status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x ib3 %16.16llX/%4.4x\n", 1178 gpu_read(gpu, REG_A8XX_RBBM_GFX_BV_STATUS), 1179 gpu_read(gpu, REG_A8XX_CP_RB_RPTR_BV), 1180 gpu_read(gpu, REG_A6XX_CP_RB_WPTR), 1181 gpu_periph_read64(gpu, CP_PERIPH_IB1_BASE_LO), 1182 gpu_periph_read(gpu, CP_PERIPH_IB1_OFFSET), 1183 gpu_periph_read64(gpu, CP_PERIPH_IB2_BASE_LO), 1184 gpu_periph_read(gpu, CP_PERIPH_IB2_OFFSET), 1185 gpu_periph_read64(gpu, CP_PERIPH_IB3_BASE_LO), 1186 gpu_periph_read(gpu, CP_PERIPH_IB3_OFFSET)); 1187 1188 a8xx_aperture_release(gpu, flags); 1189 a8xx_aperture_clear(gpu); 1190 1191 /* Turn off the hangcheck timer to keep it from bothering us */ 1192 timer_delete(&gpu->hangcheck_timer); 1193 1194 kthread_queue_work(gpu->worker, &gpu->recover_work); 1195 } 1196 1197 static void a8xx_sw_fuse_violation_irq(struct msm_gpu *gpu) 1198 { 1199 u32 status; 1200 1201 status = gpu_read(gpu, REG_A8XX_RBBM_SW_FUSE_INT_STATUS); 1202 gpu_write(gpu, REG_A8XX_RBBM_SW_FUSE_INT_MASK, 0); 1203 1204 dev_err_ratelimited(&gpu->pdev->dev, "SW fuse violation status=%8.8x\n", status); 1205 1206 /* 1207 * Ignore FASTBLEND violations, because the HW will silently fall back 1208 * to legacy blending. 1209 */ 1210 if (status & (A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING | 1211 A7XX_CX_MISC_SW_FUSE_VALUE_LPAC)) { 1212 timer_delete(&gpu->hangcheck_timer); 1213 1214 kthread_queue_work(gpu->worker, &gpu->recover_work); 1215 } 1216 } 1217 1218 irqreturn_t a8xx_irq(struct msm_gpu *gpu) 1219 { 1220 struct msm_drm_private *priv = gpu->dev->dev_private; 1221 u32 status = gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS); 1222 1223 gpu_write(gpu, REG_A8XX_RBBM_INT_CLEAR_CMD, status); 1224 1225 if (priv->disable_err_irq) { 1226 /* Turn off interrupts to avoid interrupt storm */ 1227 gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS); 1228 status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; 1229 } 1230 1231 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) 1232 a8xx_fault_detect_irq(gpu); 1233 1234 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR) { 1235 u32 rl0, rl1; 1236 1237 rl0 = gpu_read(gpu, REG_A8XX_CP_RL_ERROR_DETAILS_0); 1238 rl1 = gpu_read(gpu, REG_A8XX_CP_RL_ERROR_DETAILS_1); 1239 dev_err_ratelimited(&gpu->pdev->dev, 1240 "CP | AHB bus error RL_ERROR_0: %x, RL_ERROR_1: %x\n", rl0, rl1); 1241 } 1242 1243 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR) 1244 a8xx_cp_hw_err_irq(gpu); 1245 1246 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW) 1247 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); 1248 1249 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) 1250 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); 1251 1252 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) 1253 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); 1254 1255 if (status & A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR) 1256 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Trap interrupt\n"); 1257 1258 if (status & A6XX_RBBM_INT_0_MASK_SWFUSEVIOLATION) 1259 a8xx_sw_fuse_violation_irq(gpu); 1260 1261 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1262 msm_gpu_retire(gpu); 1263 a8xx_preempt_trigger(gpu); 1264 } 1265 1266 if (status & A6XX_RBBM_INT_0_MASK_CP_SW) 1267 a8xx_preempt_irq(gpu); 1268 1269 return IRQ_HANDLED; 1270 } 1271 1272 void a8xx_llc_activate(struct a6xx_gpu *a6xx_gpu) 1273 { 1274 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 1275 struct msm_gpu *gpu = &adreno_gpu->base; 1276 1277 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { 1278 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); 1279 1280 gpu_scid &= GENMASK(5, 0); 1281 1282 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, 1283 FIELD_PREP(GENMASK(29, 24), gpu_scid) | 1284 FIELD_PREP(GENMASK(23, 18), gpu_scid) | 1285 FIELD_PREP(GENMASK(17, 12), gpu_scid) | 1286 FIELD_PREP(GENMASK(11, 6), gpu_scid) | 1287 FIELD_PREP(GENMASK(5, 0), gpu_scid)); 1288 1289 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, 1290 FIELD_PREP(GENMASK(27, 22), gpu_scid) | 1291 FIELD_PREP(GENMASK(21, 16), gpu_scid) | 1292 FIELD_PREP(GENMASK(15, 10), gpu_scid) | 1293 BIT(8)); 1294 } 1295 1296 llcc_slice_activate(a6xx_gpu->htw_llc_slice); 1297 } 1298 1299 #define GBIF_CLIENT_HALT_MASK BIT(0) 1300 #define GBIF_ARB_HALT_MASK BIT(1) 1301 #define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0) 1302 #define VBIF_RESET_ACK_MASK 0xF0 1303 #define GPR0_GBIF_HALT_REQUEST 0x1E0 1304 1305 void a8xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off) 1306 { 1307 struct msm_gpu *gpu = &adreno_gpu->base; 1308 1309 if (gx_off) { 1310 /* Halt the gx side of GBIF */ 1311 gpu_write(gpu, REG_A8XX_RBBM_GBIF_HALT, 1); 1312 spin_until(gpu_read(gpu, REG_A8XX_RBBM_GBIF_HALT_ACK) & 1); 1313 } 1314 1315 /* Halt new client requests on GBIF */ 1316 gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); 1317 spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & 1318 (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); 1319 1320 /* Halt all AXI requests on GBIF */ 1321 gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); 1322 spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & 1323 (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); 1324 1325 /* The GBIF halt needs to be explicitly cleared */ 1326 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); 1327 } 1328 1329 u64 a8xx_gmu_get_timestamp(struct msm_gpu *gpu) 1330 { 1331 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1332 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 1333 u64 count_hi, count_lo, temp; 1334 1335 do { 1336 count_hi = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H); 1337 count_lo = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_L); 1338 temp = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H); 1339 } while (unlikely(count_hi != temp)); 1340 1341 return (count_hi << 32) | count_lo; 1342 } 1343 1344 u64 a8xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) 1345 { 1346 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 1347 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 1348 u64 busy_cycles; 1349 1350 /* 19.2MHz */ 1351 *out_sample_rate = 19200000; 1352 1353 busy_cycles = gmu_read64(&a6xx_gpu->gmu, 1354 REG_A8XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, 1355 REG_A8XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H); 1356 1357 return busy_cycles; 1358 } 1359 1360 bool a8xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 1361 { 1362 return true; 1363 } 1364 1365 void a8xx_perfcntr_flush(struct msm_gpu *gpu) 1366 { 1367 u32 val; 1368 1369 /* 1370 * Flush delta counters (both perf counters and pipe stats) present in 1371 * RBBM_S and RBBM_US to perf RAM logic to get the latest data. 1372 */ 1373 gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_FLUSH_HOST_CMD, BIT(0)); 1374 gpu_write(gpu, REG_A8XX_RBBM_SLICE_PERFCTR_FLUSH_HOST_CMD, BIT(0)); 1375 1376 /* Ensure all writes are posted before polling status register */ 1377 wmb(); 1378 1379 if (gpu_poll_timeout(gpu, REG_A8XX_RBBM_PERFCTR_FLUSH_HOST_STATUS, val, 1380 val & BIT(0), 100, 100 * 1000)) { 1381 dev_err(&gpu->pdev->dev, "Perfcounter flush timed out: status=0x%08x\n", val); 1382 } 1383 } 1384