1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, The Linux Foundation. All rights reserved. */ 3 /* Copyright (c) 2023 Collabora, Ltd. */ 4 /* Copyright (c) 2024 Valve Corporation */ 5 6 #include "msm_gem.h" 7 #include "a6xx_gpu.h" 8 #include "a6xx_gmu.xml.h" 9 #include "a6xx_preempt.h" 10 #include "msm_mmu.h" 11 #include "msm_gpu_trace.h" 12 13 static void a6xx_preempt_timer(struct timer_list *t) 14 { 15 struct a6xx_gpu *a6xx_gpu = timer_container_of(a6xx_gpu, t, 16 preempt_timer); 17 struct msm_gpu *gpu = &a6xx_gpu->base.base; 18 struct drm_device *dev = gpu->dev; 19 20 if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 21 return; 22 23 dev_err(dev->dev, "%s: preemption timed out\n", gpu->name); 24 kthread_queue_work(gpu->worker, &gpu->recover_work); 25 } 26 27 static void preempt_prepare_postamble(struct a6xx_gpu *a6xx_gpu) 28 { 29 u32 *postamble = a6xx_gpu->preempt_postamble_ptr; 30 u32 count = 0; 31 32 postamble[count++] = PKT7(CP_REG_RMW, 3); 33 postamble[count++] = REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD; 34 postamble[count++] = 0; 35 postamble[count++] = 1; 36 37 postamble[count++] = PKT7(CP_WAIT_REG_MEM, 6); 38 postamble[count++] = CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ); 39 postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_LO( 40 REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS); 41 postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_HI(0); 42 postamble[count++] = CP_WAIT_REG_MEM_3_REF(0x1); 43 postamble[count++] = CP_WAIT_REG_MEM_4_MASK(0x1); 44 postamble[count++] = CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0); 45 46 a6xx_gpu->preempt_postamble_len = count; 47 48 a6xx_gpu->postamble_enabled = true; 49 } 50 51 static void preempt_disable_postamble(struct a6xx_gpu *a6xx_gpu) 52 { 53 u32 *postamble = a6xx_gpu->preempt_postamble_ptr; 54 55 /* 56 * Disable the postamble by replacing the first packet header with a NOP 57 * that covers the whole buffer. 58 */ 59 *postamble = PKT7(CP_NOP, (a6xx_gpu->preempt_postamble_len - 1)); 60 61 a6xx_gpu->postamble_enabled = false; 62 } 63 64 /* 65 * Set preemption keepalive vote. Please note that this vote is different from the one used in 66 * a6xx_irq() 67 */ 68 static void a6xx_preempt_keepalive_vote(struct msm_gpu *gpu, bool on) 69 { 70 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 71 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 72 73 if (adreno_has_gmu_wrapper(adreno_gpu)) 74 return; 75 76 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_PWR_COL_PREEMPT_KEEPALIVE, on); 77 } 78 79 void a6xx_preempt_irq(struct msm_gpu *gpu) 80 { 81 uint32_t status; 82 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 83 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 84 struct drm_device *dev = gpu->dev; 85 86 if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 87 return; 88 89 /* Delete the preemption watchdog timer */ 90 timer_delete(&a6xx_gpu->preempt_timer); 91 92 /* 93 * The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL 94 * to zero before firing the interrupt, but there is a non zero chance 95 * of a hardware condition or a software race that could set it again 96 * before we have a chance to finish. If that happens, log and go for 97 * recovery 98 */ 99 status = gpu_read(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL); 100 if (unlikely(status & A6XX_CP_CONTEXT_SWITCH_CNTL_STOP)) { 101 DRM_DEV_ERROR(&gpu->pdev->dev, 102 "!!!!!!!!!!!!!!!! preemption faulted !!!!!!!!!!!!!! irq\n"); 103 set_preempt_state(a6xx_gpu, PREEMPT_FAULTED); 104 dev_err(dev->dev, "%s: Preemption failed to complete\n", 105 gpu->name); 106 kthread_queue_work(gpu->worker, &gpu->recover_work); 107 return; 108 } 109 110 a6xx_gpu->cur_ring = a6xx_gpu->next_ring; 111 a6xx_gpu->next_ring = NULL; 112 113 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 114 115 update_wptr(a6xx_gpu, a6xx_gpu->cur_ring); 116 117 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 118 119 a6xx_preempt_keepalive_vote(gpu, false); 120 121 trace_msm_gpu_preemption_irq(a6xx_gpu->cur_ring->id); 122 123 /* 124 * Retrigger preemption to avoid a deadlock that might occur when preemption 125 * is skipped due to it being already in flight when requested. 126 */ 127 a6xx_preempt_trigger(gpu); 128 } 129 130 void a6xx_preempt_hw_init(struct msm_gpu *gpu) 131 { 132 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 133 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 134 int i; 135 136 /* No preemption if we only have one ring */ 137 if (gpu->nr_rings == 1) 138 return; 139 140 for (i = 0; i < gpu->nr_rings; i++) { 141 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[i]; 142 143 record_ptr->wptr = 0; 144 record_ptr->rptr = 0; 145 record_ptr->rptr_addr = shadowptr(a6xx_gpu, gpu->rb[i]); 146 record_ptr->info = 0; 147 record_ptr->data = 0; 148 record_ptr->rbase = gpu->rb[i]->iova; 149 } 150 151 /* Write a 0 to signal that we aren't switching pagetables */ 152 gpu_write64(gpu, REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, 0); 153 154 /* Enable the GMEM save/restore feature for preemption */ 155 gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE, 0x1); 156 157 /* Reset the preemption state */ 158 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 159 160 spin_lock_init(&a6xx_gpu->eval_lock); 161 162 /* Always come up on rb 0 */ 163 a6xx_gpu->cur_ring = gpu->rb[0]; 164 } 165 166 void a6xx_preempt_trigger(struct msm_gpu *gpu) 167 { 168 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 169 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 170 unsigned long flags; 171 struct msm_ringbuffer *ring; 172 unsigned int cntl; 173 bool sysprof; 174 175 if (gpu->nr_rings == 1) 176 return; 177 178 /* 179 * Lock to make sure another thread attempting preemption doesn't skip it 180 * while we are still evaluating the next ring. This makes sure the other 181 * thread does start preemption if we abort it and avoids a soft lock. 182 */ 183 spin_lock_irqsave(&a6xx_gpu->eval_lock, flags); 184 185 /* 186 * Try to start preemption by moving from NONE to START. If 187 * unsuccessful, a preemption is already in flight 188 */ 189 if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START)) { 190 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 191 return; 192 } 193 194 cntl = A6XX_CP_CONTEXT_SWITCH_CNTL_LEVEL(a6xx_gpu->preempt_level); 195 196 if (a6xx_gpu->skip_save_restore) 197 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_SKIP_SAVE_RESTORE; 198 199 if (a6xx_gpu->uses_gmem) 200 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_USES_GMEM; 201 202 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_STOP; 203 204 /* Get the next ring to preempt to */ 205 ring = get_next_ring(gpu); 206 207 /* 208 * If no ring is populated or the highest priority ring is the current 209 * one do nothing except to update the wptr to the latest and greatest 210 */ 211 if (!ring || (a6xx_gpu->cur_ring == ring)) { 212 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 213 update_wptr(a6xx_gpu, a6xx_gpu->cur_ring); 214 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 215 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 216 return; 217 } 218 219 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 220 221 spin_lock_irqsave(&ring->preempt_lock, flags); 222 223 struct a7xx_cp_smmu_info *smmu_info_ptr = 224 a6xx_gpu->preempt_smmu[ring->id]; 225 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[ring->id]; 226 u64 ttbr0 = ring->memptrs->ttbr0; 227 u32 context_idr = ring->memptrs->context_idr; 228 229 smmu_info_ptr->ttbr0 = ttbr0; 230 smmu_info_ptr->context_idr = context_idr; 231 record_ptr->wptr = get_wptr(ring); 232 233 /* 234 * The GPU will write the wptr we set above when we preempt. Reset 235 * restore_wptr to make sure that we don't write WPTR to the same 236 * thing twice. It's still possible subsequent submissions will update 237 * wptr again, in which case they will set the flag to true. This has 238 * to be protected by the lock for setting the flag and updating wptr 239 * to be atomic. 240 */ 241 ring->restore_wptr = false; 242 243 trace_msm_gpu_preemption_trigger(a6xx_gpu->cur_ring->id, ring->id); 244 245 spin_unlock_irqrestore(&ring->preempt_lock, flags); 246 247 /* Set the keepalive bit to keep the GPU ON until preemption is complete */ 248 a6xx_preempt_keepalive_vote(gpu, true); 249 250 a6xx_fenced_write(a6xx_gpu, 251 REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, a6xx_gpu->preempt_smmu_iova[ring->id], 252 BIT(1), true); 253 254 a6xx_fenced_write(a6xx_gpu, 255 REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR, 256 a6xx_gpu->preempt_iova[ring->id], BIT(1), true); 257 258 a6xx_gpu->next_ring = ring; 259 260 /* Start a timer to catch a stuck preemption */ 261 mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 262 263 /* Enable or disable postamble as needed */ 264 sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1; 265 266 if (!sysprof && !a6xx_gpu->postamble_enabled) 267 preempt_prepare_postamble(a6xx_gpu); 268 269 if (sysprof && a6xx_gpu->postamble_enabled) 270 preempt_disable_postamble(a6xx_gpu); 271 272 /* Set the preemption state to triggered */ 273 set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED); 274 275 /* Trigger the preemption */ 276 a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL, cntl, BIT(1), false); 277 } 278 279 static int preempt_init_ring(struct a6xx_gpu *a6xx_gpu, 280 struct msm_ringbuffer *ring) 281 { 282 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 283 struct msm_gpu *gpu = &adreno_gpu->base; 284 struct drm_gem_object *bo = NULL; 285 phys_addr_t ttbr; 286 u64 iova = 0; 287 void *ptr; 288 int asid; 289 290 ptr = msm_gem_kernel_new(gpu->dev, 291 PREEMPT_RECORD_SIZE(adreno_gpu), 292 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->vm, &bo, &iova); 293 294 if (IS_ERR(ptr)) 295 return PTR_ERR(ptr); 296 297 memset(ptr, 0, PREEMPT_RECORD_SIZE(adreno_gpu)); 298 299 msm_gem_object_set_name(bo, "preempt_record ring%d", ring->id); 300 301 a6xx_gpu->preempt_bo[ring->id] = bo; 302 a6xx_gpu->preempt_iova[ring->id] = iova; 303 a6xx_gpu->preempt[ring->id] = ptr; 304 305 struct a6xx_preempt_record *record_ptr = ptr; 306 307 ptr = msm_gem_kernel_new(gpu->dev, 308 PREEMPT_SMMU_INFO_SIZE, 309 MSM_BO_WC | MSM_BO_MAP_PRIV | MSM_BO_GPU_READONLY, 310 gpu->vm, &bo, &iova); 311 312 if (IS_ERR(ptr)) 313 return PTR_ERR(ptr); 314 315 memset(ptr, 0, PREEMPT_SMMU_INFO_SIZE); 316 317 msm_gem_object_set_name(bo, "preempt_smmu_info ring%d", ring->id); 318 319 a6xx_gpu->preempt_smmu_bo[ring->id] = bo; 320 a6xx_gpu->preempt_smmu_iova[ring->id] = iova; 321 a6xx_gpu->preempt_smmu[ring->id] = ptr; 322 323 struct a7xx_cp_smmu_info *smmu_info_ptr = ptr; 324 325 msm_iommu_pagetable_params(to_msm_vm(gpu->vm)->mmu, &ttbr, &asid); 326 327 smmu_info_ptr->magic = GEN7_CP_SMMU_INFO_MAGIC; 328 smmu_info_ptr->ttbr0 = ttbr; 329 smmu_info_ptr->asid = 0xdecafbad; 330 smmu_info_ptr->context_idr = 0; 331 332 /* Set up the defaults on the preemption record */ 333 record_ptr->magic = A6XX_PREEMPT_RECORD_MAGIC; 334 record_ptr->info = 0; 335 record_ptr->data = 0; 336 record_ptr->rptr = 0; 337 record_ptr->wptr = 0; 338 record_ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; 339 record_ptr->rbase = ring->iova; 340 record_ptr->counter = 0; 341 record_ptr->bv_rptr_addr = rbmemptr(ring, bv_rptr); 342 343 return 0; 344 } 345 346 void a6xx_preempt_fini(struct msm_gpu *gpu) 347 { 348 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 349 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 350 int i; 351 352 for (i = 0; i < gpu->nr_rings; i++) 353 msm_gem_kernel_put(a6xx_gpu->preempt_bo[i], gpu->vm); 354 } 355 356 void a6xx_preempt_init(struct msm_gpu *gpu) 357 { 358 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 359 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 360 int i; 361 362 /* No preemption if we only have one ring */ 363 if (gpu->nr_rings <= 1) 364 return; 365 366 for (i = 0; i < gpu->nr_rings; i++) { 367 if (preempt_init_ring(a6xx_gpu, gpu->rb[i])) 368 goto fail; 369 } 370 371 /* TODO: make this configurable? */ 372 a6xx_gpu->preempt_level = 1; 373 a6xx_gpu->uses_gmem = 1; 374 a6xx_gpu->skip_save_restore = 1; 375 376 a6xx_gpu->preempt_postamble_ptr = msm_gem_kernel_new(gpu->dev, 377 PAGE_SIZE, 378 MSM_BO_WC | MSM_BO_MAP_PRIV | MSM_BO_GPU_READONLY, 379 gpu->vm, &a6xx_gpu->preempt_postamble_bo, 380 &a6xx_gpu->preempt_postamble_iova); 381 382 if (IS_ERR(a6xx_gpu->preempt_postamble_ptr)) 383 goto fail; 384 385 preempt_prepare_postamble(a6xx_gpu); 386 387 timer_setup(&a6xx_gpu->preempt_timer, a6xx_preempt_timer, 0); 388 389 return; 390 fail: 391 /* 392 * On any failure our adventure is over. Clean up and 393 * set nr_rings to 1 to force preemption off 394 */ 395 a6xx_preempt_fini(gpu); 396 gpu->nr_rings = 1; 397 398 DRM_DEV_ERROR(&gpu->pdev->dev, 399 "preemption init failed, disabling preemption\n"); 400 401 return; 402 } 403