1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
3 */
4
5 #include "msm_gem.h"
6 #include "a5xx_gpu.h"
7
8 /*
9 * Try to transition the preemption state from old to new. Return
10 * true on success or false if the original state wasn't 'old'
11 */
try_preempt_state(struct a5xx_gpu * a5xx_gpu,enum preempt_state old,enum preempt_state new)12 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
13 enum preempt_state old, enum preempt_state new)
14 {
15 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
16 old, new);
17
18 return (cur == old);
19 }
20
21 /*
22 * Force the preemption state to the specified state. This is used in cases
23 * where the current state is known and won't change
24 */
set_preempt_state(struct a5xx_gpu * gpu,enum preempt_state new)25 static inline void set_preempt_state(struct a5xx_gpu *gpu,
26 enum preempt_state new)
27 {
28 /*
29 * preempt_state may be read by other cores trying to trigger a
30 * preemption or in the interrupt handler so barriers are needed
31 * before...
32 */
33 smp_mb__before_atomic();
34 atomic_set(&gpu->preempt_state, new);
35 /* ... and after*/
36 smp_mb__after_atomic();
37 }
38
39 /* Write the most recent wptr for the given ring into the hardware */
update_wptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
41 {
42 unsigned long flags;
43 uint32_t wptr;
44
45 if (!ring)
46 return;
47
48 spin_lock_irqsave(&ring->preempt_lock, flags);
49 wptr = get_wptr(ring);
50 spin_unlock_irqrestore(&ring->preempt_lock, flags);
51
52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
53 }
54
55 /* Return the highest priority ringbuffer with something in it */
get_next_ring(struct msm_gpu * gpu)56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
57 {
58 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
59 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
60 unsigned long flags;
61 int i;
62
63 for (i = 0; i < gpu->nr_rings; i++) {
64 bool empty;
65 struct msm_ringbuffer *ring = gpu->rb[i];
66
67 spin_lock_irqsave(&ring->preempt_lock, flags);
68 empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
69 if (!empty && ring == a5xx_gpu->cur_ring)
70 empty = ring->memptrs->fence == a5xx_gpu->last_seqno[i];
71 spin_unlock_irqrestore(&ring->preempt_lock, flags);
72
73 if (!empty)
74 return ring;
75 }
76
77 return NULL;
78 }
79
a5xx_preempt_timer(struct timer_list * t)80 static void a5xx_preempt_timer(struct timer_list *t)
81 {
82 struct a5xx_gpu *a5xx_gpu = timer_container_of(a5xx_gpu, t,
83 preempt_timer);
84 struct msm_gpu *gpu = &a5xx_gpu->base.base;
85 struct drm_device *dev = gpu->dev;
86
87 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
88 return;
89
90 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
91 kthread_queue_work(gpu->worker, &gpu->recover_work);
92 }
93
94 /* Try to trigger a preemption switch */
a5xx_preempt_trigger(struct msm_gpu * gpu)95 void a5xx_preempt_trigger(struct msm_gpu *gpu)
96 {
97 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
98 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
99 unsigned long flags;
100 struct msm_ringbuffer *ring;
101
102 if (gpu->nr_rings == 1)
103 return;
104
105 /*
106 * Serialize preemption start to ensure that we always make
107 * decision on latest state. Otherwise we can get stuck in
108 * lower priority or empty ring.
109 */
110 spin_lock_irqsave(&a5xx_gpu->preempt_start_lock, flags);
111
112 /*
113 * Try to start preemption by moving from NONE to START. If
114 * unsuccessful, a preemption is already in flight
115 */
116 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
117 goto out;
118
119 /* Get the next ring to preempt to */
120 ring = get_next_ring(gpu);
121
122 /*
123 * If no ring is populated or the highest priority ring is the current
124 * one do nothing except to update the wptr to the latest and greatest
125 */
126 if (!ring || (a5xx_gpu->cur_ring == ring)) {
127 /*
128 * Its possible that while a preemption request is in progress
129 * from an irq context, a user context trying to submit might
130 * fail to update the write pointer, because it determines
131 * that the preempt state is not PREEMPT_NONE.
132 *
133 * Close the race by introducing an intermediate
134 * state PREEMPT_ABORT to let the submit path
135 * know that the ringbuffer is not going to change
136 * and can safely update the write pointer.
137 */
138
139 set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
140 update_wptr(gpu, a5xx_gpu->cur_ring);
141 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
142 goto out;
143 }
144
145 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);
146
147 /* Make sure the wptr doesn't update while we're in motion */
148 spin_lock_irqsave(&ring->preempt_lock, flags);
149 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
150 spin_unlock_irqrestore(&ring->preempt_lock, flags);
151
152 /* Set the address of the incoming preemption record */
153 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
154 a5xx_gpu->preempt_iova[ring->id]);
155
156 a5xx_gpu->next_ring = ring;
157
158 /* Start a timer to catch a stuck preemption */
159 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
160
161 /* Set the preemption state to triggered */
162 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
163
164 /* Make sure everything is written before hitting the button */
165 wmb();
166
167 /* And actually start the preemption */
168 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
169 return;
170
171 out:
172 spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);
173 }
174
a5xx_preempt_irq(struct msm_gpu * gpu)175 void a5xx_preempt_irq(struct msm_gpu *gpu)
176 {
177 uint32_t status;
178 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
179 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
180 struct drm_device *dev = gpu->dev;
181
182 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
183 return;
184
185 /* Delete the preemption watchdog timer */
186 timer_delete(&a5xx_gpu->preempt_timer);
187
188 /*
189 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
190 * firing the interrupt, but there is a non zero chance of a hardware
191 * condition or a software race that could set it again before we have a
192 * chance to finish. If that happens, log and go for recovery
193 */
194 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
195 if (unlikely(status)) {
196 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
197 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
198 gpu->name);
199 kthread_queue_work(gpu->worker, &gpu->recover_work);
200 return;
201 }
202
203 a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
204 a5xx_gpu->next_ring = NULL;
205
206 update_wptr(gpu, a5xx_gpu->cur_ring);
207
208 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
209
210 /*
211 * Try to trigger preemption again in case there was a submit or
212 * retire during ring switch
213 */
214 a5xx_preempt_trigger(gpu);
215 }
216
a5xx_preempt_hw_init(struct msm_gpu * gpu)217 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
218 {
219 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
220 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
221 int i;
222
223 /* Always come up on rb 0 */
224 a5xx_gpu->cur_ring = gpu->rb[0];
225
226 /* No preemption if we only have one ring */
227 if (gpu->nr_rings == 1)
228 return;
229
230 for (i = 0; i < gpu->nr_rings; i++) {
231 a5xx_gpu->preempt[i]->data = 0;
232 a5xx_gpu->preempt[i]->info = 0;
233 a5xx_gpu->preempt[i]->wptr = 0;
234 a5xx_gpu->preempt[i]->rptr = 0;
235 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
236 a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);
237 }
238
239 /* Write a 0 to signal that we aren't switching pagetables */
240 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);
241
242 /* Reset the preemption state */
243 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
244 }
245
preempt_init_ring(struct a5xx_gpu * a5xx_gpu,struct msm_ringbuffer * ring)246 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
247 struct msm_ringbuffer *ring)
248 {
249 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
250 struct msm_gpu *gpu = &adreno_gpu->base;
251 struct a5xx_preempt_record *ptr;
252 void *counters;
253 struct drm_gem_object *bo = NULL, *counters_bo = NULL;
254 u64 iova = 0, counters_iova = 0;
255
256 ptr = msm_gem_kernel_new(gpu->dev,
257 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
258 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);
259
260 if (IS_ERR(ptr))
261 return PTR_ERR(ptr);
262
263 /* The buffer to store counters needs to be unprivileged */
264 counters = msm_gem_kernel_new(gpu->dev,
265 A5XX_PREEMPT_COUNTER_SIZE,
266 MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);
267 if (IS_ERR(counters)) {
268 msm_gem_kernel_put(bo, gpu->aspace);
269 return PTR_ERR(counters);
270 }
271
272 msm_gem_object_set_name(bo, "preempt");
273 msm_gem_object_set_name(counters_bo, "preempt_counters");
274
275 a5xx_gpu->preempt_bo[ring->id] = bo;
276 a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
277 a5xx_gpu->preempt_iova[ring->id] = iova;
278 a5xx_gpu->preempt[ring->id] = ptr;
279
280 /* Set up the defaults on the preemption record */
281
282 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
283 ptr->info = 0;
284 ptr->data = 0;
285 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
286
287 ptr->counter = counters_iova;
288
289 return 0;
290 }
291
a5xx_preempt_fini(struct msm_gpu * gpu)292 void a5xx_preempt_fini(struct msm_gpu *gpu)
293 {
294 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
295 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
296 int i;
297
298 for (i = 0; i < gpu->nr_rings; i++) {
299 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);
300 msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);
301 }
302 }
303
a5xx_preempt_init(struct msm_gpu * gpu)304 void a5xx_preempt_init(struct msm_gpu *gpu)
305 {
306 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
307 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
308 int i;
309
310 /* No preemption if we only have one ring */
311 if (gpu->nr_rings <= 1)
312 return;
313
314 for (i = 0; i < gpu->nr_rings; i++) {
315 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
316 /*
317 * On any failure our adventure is over. Clean up and
318 * set nr_rings to 1 to force preemption off
319 */
320 a5xx_preempt_fini(gpu);
321 gpu->nr_rings = 1;
322
323 return;
324 }
325 }
326
327 spin_lock_init(&a5xx_gpu->preempt_start_lock);
328 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
329 }
330