1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, The Linux Foundation. All rights reserved. */ 3 /* Copyright (c) 2023 Collabora, Ltd. */ 4 /* Copyright (c) 2024 Valve Corporation */ 5 6 #include "msm_gem.h" 7 #include "a6xx_gpu.h" 8 #include "a6xx_gmu.xml.h" 9 #include "msm_mmu.h" 10 11 /* 12 * Try to transition the preemption state from old to new. Return 13 * true on success or false if the original state wasn't 'old' 14 */ 15 static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu, 16 enum a6xx_preempt_state old, enum a6xx_preempt_state new) 17 { 18 enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state, 19 old, new); 20 21 return (cur == old); 22 } 23 24 /* 25 * Force the preemption state to the specified state. This is used in cases 26 * where the current state is known and won't change 27 */ 28 static inline void set_preempt_state(struct a6xx_gpu *gpu, 29 enum a6xx_preempt_state new) 30 { 31 /* 32 * preempt_state may be read by other cores trying to trigger a 33 * preemption or in the interrupt handler so barriers are needed 34 * before... 35 */ 36 smp_mb__before_atomic(); 37 atomic_set(&gpu->preempt_state, new); 38 /* ... and after*/ 39 smp_mb__after_atomic(); 40 } 41 42 /* Write the most recent wptr for the given ring into the hardware */ 43 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 44 { 45 unsigned long flags; 46 uint32_t wptr; 47 48 spin_lock_irqsave(&ring->preempt_lock, flags); 49 50 if (ring->restore_wptr) { 51 wptr = get_wptr(ring); 52 53 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); 54 55 ring->restore_wptr = false; 56 } 57 58 spin_unlock_irqrestore(&ring->preempt_lock, flags); 59 } 60 61 /* Return the highest priority ringbuffer with something in it */ 62 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) 63 { 64 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 65 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 66 67 unsigned long flags; 68 int i; 69 70 for (i = 0; i < gpu->nr_rings; i++) { 71 bool empty; 72 struct msm_ringbuffer *ring = gpu->rb[i]; 73 74 spin_lock_irqsave(&ring->preempt_lock, flags); 75 empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring)); 76 if (!empty && ring == a6xx_gpu->cur_ring) 77 empty = ring->memptrs->fence == a6xx_gpu->last_seqno[i]; 78 spin_unlock_irqrestore(&ring->preempt_lock, flags); 79 80 if (!empty) 81 return ring; 82 } 83 84 return NULL; 85 } 86 87 static void a6xx_preempt_timer(struct timer_list *t) 88 { 89 struct a6xx_gpu *a6xx_gpu = from_timer(a6xx_gpu, t, preempt_timer); 90 struct msm_gpu *gpu = &a6xx_gpu->base.base; 91 struct drm_device *dev = gpu->dev; 92 93 if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 94 return; 95 96 dev_err(dev->dev, "%s: preemption timed out\n", gpu->name); 97 kthread_queue_work(gpu->worker, &gpu->recover_work); 98 } 99 100 void a6xx_preempt_irq(struct msm_gpu *gpu) 101 { 102 uint32_t status; 103 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 104 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 105 struct drm_device *dev = gpu->dev; 106 107 if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 108 return; 109 110 /* Delete the preemption watchdog timer */ 111 del_timer(&a6xx_gpu->preempt_timer); 112 113 /* 114 * The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL 115 * to zero before firing the interrupt, but there is a non zero chance 116 * of a hardware condition or a software race that could set it again 117 * before we have a chance to finish. If that happens, log and go for 118 * recovery 119 */ 120 status = gpu_read(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL); 121 if (unlikely(status & A6XX_CP_CONTEXT_SWITCH_CNTL_STOP)) { 122 DRM_DEV_ERROR(&gpu->pdev->dev, 123 "!!!!!!!!!!!!!!!! preemption faulted !!!!!!!!!!!!!! irq\n"); 124 set_preempt_state(a6xx_gpu, PREEMPT_FAULTED); 125 dev_err(dev->dev, "%s: Preemption failed to complete\n", 126 gpu->name); 127 kthread_queue_work(gpu->worker, &gpu->recover_work); 128 return; 129 } 130 131 a6xx_gpu->cur_ring = a6xx_gpu->next_ring; 132 a6xx_gpu->next_ring = NULL; 133 134 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 135 136 update_wptr(gpu, a6xx_gpu->cur_ring); 137 138 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 139 140 /* 141 * Retrigger preemption to avoid a deadlock that might occur when preemption 142 * is skipped due to it being already in flight when requested. 143 */ 144 a6xx_preempt_trigger(gpu); 145 } 146 147 void a6xx_preempt_hw_init(struct msm_gpu *gpu) 148 { 149 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 150 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 151 int i; 152 153 /* No preemption if we only have one ring */ 154 if (gpu->nr_rings == 1) 155 return; 156 157 for (i = 0; i < gpu->nr_rings; i++) { 158 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[i]; 159 160 record_ptr->wptr = 0; 161 record_ptr->rptr = 0; 162 record_ptr->rptr_addr = shadowptr(a6xx_gpu, gpu->rb[i]); 163 record_ptr->info = 0; 164 record_ptr->data = 0; 165 record_ptr->rbase = gpu->rb[i]->iova; 166 } 167 168 /* Write a 0 to signal that we aren't switching pagetables */ 169 gpu_write64(gpu, REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, 0); 170 171 /* Enable the GMEM save/restore feature for preemption */ 172 gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, 0x1); 173 174 /* Reset the preemption state */ 175 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 176 177 spin_lock_init(&a6xx_gpu->eval_lock); 178 179 /* Always come up on rb 0 */ 180 a6xx_gpu->cur_ring = gpu->rb[0]; 181 } 182 183 void a6xx_preempt_trigger(struct msm_gpu *gpu) 184 { 185 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 186 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 187 unsigned long flags; 188 struct msm_ringbuffer *ring; 189 unsigned int cntl; 190 191 if (gpu->nr_rings == 1) 192 return; 193 194 /* 195 * Lock to make sure another thread attempting preemption doesn't skip it 196 * while we are still evaluating the next ring. This makes sure the other 197 * thread does start preemption if we abort it and avoids a soft lock. 198 */ 199 spin_lock_irqsave(&a6xx_gpu->eval_lock, flags); 200 201 /* 202 * Try to start preemption by moving from NONE to START. If 203 * unsuccessful, a preemption is already in flight 204 */ 205 if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START)) { 206 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 207 return; 208 } 209 210 cntl = A6XX_CP_CONTEXT_SWITCH_CNTL_LEVEL(a6xx_gpu->preempt_level); 211 212 if (a6xx_gpu->skip_save_restore) 213 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_SKIP_SAVE_RESTORE; 214 215 if (a6xx_gpu->uses_gmem) 216 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_USES_GMEM; 217 218 cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_STOP; 219 220 /* Get the next ring to preempt to */ 221 ring = get_next_ring(gpu); 222 223 /* 224 * If no ring is populated or the highest priority ring is the current 225 * one do nothing except to update the wptr to the latest and greatest 226 */ 227 if (!ring || (a6xx_gpu->cur_ring == ring)) { 228 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 229 update_wptr(gpu, a6xx_gpu->cur_ring); 230 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 231 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 232 return; 233 } 234 235 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 236 237 spin_lock_irqsave(&ring->preempt_lock, flags); 238 239 struct a7xx_cp_smmu_info *smmu_info_ptr = 240 a6xx_gpu->preempt_smmu[ring->id]; 241 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[ring->id]; 242 u64 ttbr0 = ring->memptrs->ttbr0; 243 u32 context_idr = ring->memptrs->context_idr; 244 245 smmu_info_ptr->ttbr0 = ttbr0; 246 smmu_info_ptr->context_idr = context_idr; 247 record_ptr->wptr = get_wptr(ring); 248 249 /* 250 * The GPU will write the wptr we set above when we preempt. Reset 251 * restore_wptr to make sure that we don't write WPTR to the same 252 * thing twice. It's still possible subsequent submissions will update 253 * wptr again, in which case they will set the flag to true. This has 254 * to be protected by the lock for setting the flag and updating wptr 255 * to be atomic. 256 */ 257 ring->restore_wptr = false; 258 259 spin_unlock_irqrestore(&ring->preempt_lock, flags); 260 261 gpu_write64(gpu, 262 REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, 263 a6xx_gpu->preempt_smmu_iova[ring->id]); 264 265 gpu_write64(gpu, 266 REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR, 267 a6xx_gpu->preempt_iova[ring->id]); 268 269 a6xx_gpu->next_ring = ring; 270 271 /* Start a timer to catch a stuck preemption */ 272 mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 273 274 /* Set the preemption state to triggered */ 275 set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED); 276 277 /* Trigger the preemption */ 278 gpu_write(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL, cntl); 279 } 280 281 static int preempt_init_ring(struct a6xx_gpu *a6xx_gpu, 282 struct msm_ringbuffer *ring) 283 { 284 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 285 struct msm_gpu *gpu = &adreno_gpu->base; 286 struct drm_gem_object *bo = NULL; 287 phys_addr_t ttbr; 288 u64 iova = 0; 289 void *ptr; 290 int asid; 291 292 ptr = msm_gem_kernel_new(gpu->dev, 293 PREEMPT_RECORD_SIZE(adreno_gpu), 294 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova); 295 296 if (IS_ERR(ptr)) 297 return PTR_ERR(ptr); 298 299 memset(ptr, 0, PREEMPT_RECORD_SIZE(adreno_gpu)); 300 301 msm_gem_object_set_name(bo, "preempt_record ring%d", ring->id); 302 303 a6xx_gpu->preempt_bo[ring->id] = bo; 304 a6xx_gpu->preempt_iova[ring->id] = iova; 305 a6xx_gpu->preempt[ring->id] = ptr; 306 307 struct a6xx_preempt_record *record_ptr = ptr; 308 309 ptr = msm_gem_kernel_new(gpu->dev, 310 PREEMPT_SMMU_INFO_SIZE, 311 MSM_BO_WC | MSM_BO_MAP_PRIV | MSM_BO_GPU_READONLY, 312 gpu->aspace, &bo, &iova); 313 314 if (IS_ERR(ptr)) 315 return PTR_ERR(ptr); 316 317 memset(ptr, 0, PREEMPT_SMMU_INFO_SIZE); 318 319 msm_gem_object_set_name(bo, "preempt_smmu_info ring%d", ring->id); 320 321 a6xx_gpu->preempt_smmu_bo[ring->id] = bo; 322 a6xx_gpu->preempt_smmu_iova[ring->id] = iova; 323 a6xx_gpu->preempt_smmu[ring->id] = ptr; 324 325 struct a7xx_cp_smmu_info *smmu_info_ptr = ptr; 326 327 msm_iommu_pagetable_params(gpu->aspace->mmu, &ttbr, &asid); 328 329 smmu_info_ptr->magic = GEN7_CP_SMMU_INFO_MAGIC; 330 smmu_info_ptr->ttbr0 = ttbr; 331 smmu_info_ptr->asid = 0xdecafbad; 332 smmu_info_ptr->context_idr = 0; 333 334 /* Set up the defaults on the preemption record */ 335 record_ptr->magic = A6XX_PREEMPT_RECORD_MAGIC; 336 record_ptr->info = 0; 337 record_ptr->data = 0; 338 record_ptr->rptr = 0; 339 record_ptr->wptr = 0; 340 record_ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; 341 record_ptr->rbase = ring->iova; 342 record_ptr->counter = 0; 343 record_ptr->bv_rptr_addr = rbmemptr(ring, bv_rptr); 344 345 return 0; 346 } 347 348 void a6xx_preempt_fini(struct msm_gpu *gpu) 349 { 350 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 351 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 352 int i; 353 354 for (i = 0; i < gpu->nr_rings; i++) 355 msm_gem_kernel_put(a6xx_gpu->preempt_bo[i], gpu->aspace); 356 } 357 358 void a6xx_preempt_init(struct msm_gpu *gpu) 359 { 360 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 361 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 362 int i; 363 364 /* No preemption if we only have one ring */ 365 if (gpu->nr_rings <= 1) 366 return; 367 368 for (i = 0; i < gpu->nr_rings; i++) { 369 if (preempt_init_ring(a6xx_gpu, gpu->rb[i])) 370 goto fail; 371 } 372 373 /* TODO: make this configurable? */ 374 a6xx_gpu->preempt_level = 1; 375 a6xx_gpu->uses_gmem = 1; 376 a6xx_gpu->skip_save_restore = 1; 377 378 timer_setup(&a6xx_gpu->preempt_timer, a6xx_preempt_timer, 0); 379 380 return; 381 fail: 382 /* 383 * On any failure our adventure is over. Clean up and 384 * set nr_rings to 1 to force preemption off 385 */ 386 a6xx_preempt_fini(gpu); 387 gpu->nr_rings = 1; 388 389 DRM_DEV_ERROR(&gpu->pdev->dev, 390 "preemption init failed, disabling preemption\n"); 391 392 return; 393 } 394