1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include "msm_gem.h" 6 #include "a5xx_gpu.h" 7 8 /* 9 * Try to transition the preemption state from old to new. Return 10 * true on success or false if the original state wasn't 'old' 11 */ 12 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu, 13 enum preempt_state old, enum preempt_state new) 14 { 15 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state, 16 old, new); 17 18 return (cur == old); 19 } 20 21 /* 22 * Force the preemption state to the specified state. This is used in cases 23 * where the current state is known and won't change 24 */ 25 static inline void set_preempt_state(struct a5xx_gpu *gpu, 26 enum preempt_state new) 27 { 28 /* 29 * preempt_state may be read by other cores trying to trigger a 30 * preemption or in the interrupt handler so barriers are needed 31 * before... 32 */ 33 smp_mb__before_atomic(); 34 atomic_set(&gpu->preempt_state, new); 35 /* ... and after*/ 36 smp_mb__after_atomic(); 37 } 38 39 /* Write the most recent wptr for the given ring into the hardware */ 40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 41 { 42 unsigned long flags; 43 uint32_t wptr; 44 45 if (!ring) 46 return; 47 48 spin_lock_irqsave(&ring->preempt_lock, flags); 49 wptr = get_wptr(ring); 50 spin_unlock_irqrestore(&ring->preempt_lock, flags); 51 52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 53 } 54 55 /* Return the highest priority ringbuffer with something in it */ 56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) 57 { 58 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 59 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 60 unsigned long flags; 61 int i; 62 63 for (i = 0; i < gpu->nr_rings; i++) { 64 bool empty; 65 struct msm_ringbuffer *ring = gpu->rb[i]; 66 67 spin_lock_irqsave(&ring->preempt_lock, flags); 68 empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring)); 69 if (!empty && ring == a5xx_gpu->cur_ring) 70 empty = ring->memptrs->fence == a5xx_gpu->last_seqno[i]; 71 spin_unlock_irqrestore(&ring->preempt_lock, flags); 72 73 if (!empty) 74 return ring; 75 } 76 77 return NULL; 78 } 79 80 static void a5xx_preempt_timer(struct timer_list *t) 81 { 82 struct a5xx_gpu *a5xx_gpu = timer_container_of(a5xx_gpu, t, 83 preempt_timer); 84 struct msm_gpu *gpu = &a5xx_gpu->base.base; 85 struct drm_device *dev = gpu->dev; 86 87 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 88 return; 89 90 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name); 91 kthread_queue_work(gpu->worker, &gpu->recover_work); 92 } 93 94 /* Try to trigger a preemption switch */ 95 void a5xx_preempt_trigger(struct msm_gpu *gpu) 96 { 97 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 98 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 99 unsigned long flags; 100 struct msm_ringbuffer *ring; 101 102 if (gpu->nr_rings == 1) 103 return; 104 105 /* 106 * Serialize preemption start to ensure that we always make 107 * decision on latest state. Otherwise we can get stuck in 108 * lower priority or empty ring. 109 */ 110 spin_lock_irqsave(&a5xx_gpu->preempt_start_lock, flags); 111 112 /* 113 * Try to start preemption by moving from NONE to START. If 114 * unsuccessful, a preemption is already in flight 115 */ 116 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START)) 117 goto out; 118 119 /* Get the next ring to preempt to */ 120 ring = get_next_ring(gpu); 121 122 /* 123 * If no ring is populated or the highest priority ring is the current 124 * one do nothing except to update the wptr to the latest and greatest 125 */ 126 if (!ring || (a5xx_gpu->cur_ring == ring)) { 127 /* 128 * Its possible that while a preemption request is in progress 129 * from an irq context, a user context trying to submit might 130 * fail to update the write pointer, because it determines 131 * that the preempt state is not PREEMPT_NONE. 132 * 133 * Close the race by introducing an intermediate 134 * state PREEMPT_ABORT to let the submit path 135 * know that the ringbuffer is not going to change 136 * and can safely update the write pointer. 137 */ 138 139 set_preempt_state(a5xx_gpu, PREEMPT_ABORT); 140 update_wptr(gpu, a5xx_gpu->cur_ring); 141 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 142 goto out; 143 } 144 145 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags); 146 147 /* Make sure the wptr doesn't update while we're in motion */ 148 spin_lock_irqsave(&ring->preempt_lock, flags); 149 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring); 150 spin_unlock_irqrestore(&ring->preempt_lock, flags); 151 152 /* Set the address of the incoming preemption record */ 153 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, 154 a5xx_gpu->preempt_iova[ring->id]); 155 156 a5xx_gpu->next_ring = ring; 157 158 /* Start a timer to catch a stuck preemption */ 159 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 160 161 /* Set the preemption state to triggered */ 162 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED); 163 164 /* Make sure everything is written before hitting the button */ 165 wmb(); 166 167 /* And actually start the preemption */ 168 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1); 169 return; 170 171 out: 172 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags); 173 } 174 175 void a5xx_preempt_irq(struct msm_gpu *gpu) 176 { 177 uint32_t status; 178 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 179 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 180 struct drm_device *dev = gpu->dev; 181 182 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 183 return; 184 185 /* Delete the preemption watchdog timer */ 186 timer_delete(&a5xx_gpu->preempt_timer); 187 188 /* 189 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before 190 * firing the interrupt, but there is a non zero chance of a hardware 191 * condition or a software race that could set it again before we have a 192 * chance to finish. If that happens, log and go for recovery 193 */ 194 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL); 195 if (unlikely(status)) { 196 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED); 197 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n", 198 gpu->name); 199 kthread_queue_work(gpu->worker, &gpu->recover_work); 200 return; 201 } 202 203 a5xx_gpu->cur_ring = a5xx_gpu->next_ring; 204 a5xx_gpu->next_ring = NULL; 205 206 update_wptr(gpu, a5xx_gpu->cur_ring); 207 208 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 209 210 /* 211 * Try to trigger preemption again in case there was a submit or 212 * retire during ring switch 213 */ 214 a5xx_preempt_trigger(gpu); 215 } 216 217 void a5xx_preempt_hw_init(struct msm_gpu *gpu) 218 { 219 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 220 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 221 int i; 222 223 /* Always come up on rb 0 */ 224 a5xx_gpu->cur_ring = gpu->rb[0]; 225 226 /* No preemption if we only have one ring */ 227 if (gpu->nr_rings == 1) 228 return; 229 230 for (i = 0; i < gpu->nr_rings; i++) { 231 a5xx_gpu->preempt[i]->data = 0; 232 a5xx_gpu->preempt[i]->info = 0; 233 a5xx_gpu->preempt[i]->wptr = 0; 234 a5xx_gpu->preempt[i]->rptr = 0; 235 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova; 236 a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]); 237 } 238 239 /* Write a 0 to signal that we aren't switching pagetables */ 240 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0); 241 242 /* Reset the preemption state */ 243 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 244 } 245 246 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, 247 struct msm_ringbuffer *ring) 248 { 249 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base; 250 struct msm_gpu *gpu = &adreno_gpu->base; 251 struct a5xx_preempt_record *ptr; 252 void *counters; 253 struct drm_gem_object *bo = NULL, *counters_bo = NULL; 254 u64 iova = 0, counters_iova = 0; 255 256 ptr = msm_gem_kernel_new(gpu->dev, 257 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE, 258 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->vm, &bo, &iova); 259 260 if (IS_ERR(ptr)) 261 return PTR_ERR(ptr); 262 263 /* The buffer to store counters needs to be unprivileged */ 264 counters = msm_gem_kernel_new(gpu->dev, 265 A5XX_PREEMPT_COUNTER_SIZE, 266 MSM_BO_WC, gpu->vm, &counters_bo, &counters_iova); 267 if (IS_ERR(counters)) { 268 msm_gem_kernel_put(bo, gpu->vm); 269 return PTR_ERR(counters); 270 } 271 272 msm_gem_object_set_name(bo, "preempt"); 273 msm_gem_object_set_name(counters_bo, "preempt_counters"); 274 275 a5xx_gpu->preempt_bo[ring->id] = bo; 276 a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo; 277 a5xx_gpu->preempt_iova[ring->id] = iova; 278 a5xx_gpu->preempt[ring->id] = ptr; 279 280 /* Set up the defaults on the preemption record */ 281 282 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC; 283 ptr->info = 0; 284 ptr->data = 0; 285 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE; 286 287 ptr->counter = counters_iova; 288 289 return 0; 290 } 291 292 void a5xx_preempt_fini(struct msm_gpu *gpu) 293 { 294 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 295 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 296 int i; 297 298 for (i = 0; i < gpu->nr_rings; i++) { 299 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->vm); 300 msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->vm); 301 } 302 } 303 304 void a5xx_preempt_init(struct msm_gpu *gpu) 305 { 306 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 307 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 308 int i; 309 310 /* No preemption if we only have one ring */ 311 if (gpu->nr_rings <= 1) 312 return; 313 314 for (i = 0; i < gpu->nr_rings; i++) { 315 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) { 316 /* 317 * On any failure our adventure is over. Clean up and 318 * set nr_rings to 1 to force preemption off 319 */ 320 a5xx_preempt_fini(gpu); 321 gpu->nr_rings = 1; 322 323 return; 324 } 325 } 326 327 spin_lock_init(&a5xx_gpu->preempt_start_lock); 328 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0); 329 } 330