1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4 /* Copyright 2019 Collabora ltd. */
5
6 #include <linux/bitfield.h>
7 #include <linux/bitmap.h>
8 #include <linux/delay.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/iopoll.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15
16 #include <drm/drm_drv.h>
17 #include <drm/drm_managed.h>
18 #include <drm/drm_print.h>
19
20 #include "panthor_device.h"
21 #include "panthor_gpu.h"
22 #include "panthor_gpu_regs.h"
23 #include "panthor_hw.h"
24
25 #define CREATE_TRACE_POINTS
26 #include "panthor_trace.h"
27
28 /**
29 * struct panthor_gpu - GPU block management data.
30 */
31 struct panthor_gpu {
32 /** @iomem: CPU mapping of GPU_CONTROL iomem region */
33 void __iomem *iomem;
34
35 /** @irq: GPU irq. */
36 struct panthor_irq irq;
37
38 /** @reqs_lock: Lock protecting access to pending_reqs. */
39 spinlock_t reqs_lock;
40
41 /** @pending_reqs: Pending GPU requests. */
42 u32 pending_reqs;
43
44 /** @reqs_acked: GPU request wait queue. */
45 wait_queue_head_t reqs_acked;
46
47 /** @cache_flush_lock: Lock to serialize cache flushes */
48 struct mutex cache_flush_lock;
49 };
50
51 #define GPU_INTERRUPTS_MASK \
52 (GPU_IRQ_FAULT | \
53 GPU_IRQ_PROTM_FAULT | \
54 GPU_IRQ_RESET_COMPLETED | \
55 GPU_IRQ_CLEAN_CACHES_COMPLETED)
56
57 #define GPU_POWER_INTERRUPTS_MASK \
58 (GPU_IRQ_POWER_CHANGED | GPU_IRQ_POWER_CHANGED_ALL)
59
panthor_gpu_coherency_set(struct panthor_device * ptdev)60 static void panthor_gpu_coherency_set(struct panthor_device *ptdev)
61 {
62 gpu_write(ptdev->gpu->iomem, GPU_COHERENCY_PROTOCOL,
63 ptdev->gpu_info.selected_coherency);
64 }
65
panthor_gpu_l2_config_set(struct panthor_device * ptdev)66 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev)
67 {
68 struct panthor_gpu *gpu = ptdev->gpu;
69 const struct panthor_soc_data *data = ptdev->soc_data;
70 u32 l2_config;
71 u32 i;
72
73 if (!data || !data->asn_hash_enable)
74 return;
75
76 if (GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id) < 11) {
77 drm_err(&ptdev->base, "Custom ASN hash not supported by the device");
78 return;
79 }
80
81 for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++)
82 gpu_write(gpu->iomem, GPU_ASN_HASH(i), data->asn_hash[i]);
83
84 l2_config = gpu_read(gpu->iomem, GPU_L2_CONFIG);
85 l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE;
86 gpu_write(gpu->iomem, GPU_L2_CONFIG, l2_config);
87 }
88
panthor_gpu_irq_handler(struct panthor_device * ptdev,u32 status)89 static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
90 {
91 struct panthor_gpu *gpu = ptdev->gpu;
92
93 gpu_write(gpu->irq.iomem, INT_CLEAR, status);
94
95 if (tracepoint_enabled(gpu_power_status) && (status & GPU_POWER_INTERRUPTS_MASK))
96 trace_gpu_power_status(ptdev->base.dev,
97 gpu_read64(gpu->iomem, SHADER_READY),
98 gpu_read64(gpu->iomem, TILER_READY),
99 gpu_read64(gpu->iomem, L2_READY));
100
101 if (status & GPU_IRQ_FAULT) {
102 u32 fault_status = gpu_read(gpu->iomem, GPU_FAULT_STATUS);
103 u64 address = gpu_read64(gpu->iomem, GPU_FAULT_ADDR);
104
105 drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n",
106 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF),
107 address);
108 }
109 if (status & GPU_IRQ_PROTM_FAULT)
110 drm_warn(&ptdev->base, "GPU Fault in protected mode\n");
111
112 spin_lock(&ptdev->gpu->reqs_lock);
113 if (status & ptdev->gpu->pending_reqs) {
114 ptdev->gpu->pending_reqs &= ~status;
115 wake_up_all(&ptdev->gpu->reqs_acked);
116 }
117 spin_unlock(&ptdev->gpu->reqs_lock);
118 }
119 PANTHOR_IRQ_HANDLER(gpu, panthor_gpu_irq_handler);
120
121 /**
122 * panthor_gpu_unplug() - Called when the GPU is unplugged.
123 * @ptdev: Device to unplug.
124 */
panthor_gpu_unplug(struct panthor_device * ptdev)125 void panthor_gpu_unplug(struct panthor_device *ptdev)
126 {
127 unsigned long flags;
128
129 /* Make sure the IRQ handler is not running after that point. */
130 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
131 panthor_gpu_irq_suspend(&ptdev->gpu->irq);
132
133 /* Wake-up all waiters. */
134 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
135 ptdev->gpu->pending_reqs = 0;
136 wake_up_all(&ptdev->gpu->reqs_acked);
137 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
138 }
139
140 /**
141 * panthor_gpu_init() - Initialize the GPU block
142 * @ptdev: Device.
143 *
144 * Return: 0 on success, a negative error code otherwise.
145 */
panthor_gpu_init(struct panthor_device * ptdev)146 int panthor_gpu_init(struct panthor_device *ptdev)
147 {
148 struct panthor_gpu *gpu;
149 u32 pa_bits;
150 int ret, irq;
151
152 gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL);
153 if (!gpu)
154 return -ENOMEM;
155
156 gpu->iomem = ptdev->iomem + GPU_CONTROL_BASE;
157 spin_lock_init(&gpu->reqs_lock);
158 init_waitqueue_head(&gpu->reqs_acked);
159 mutex_init(&gpu->cache_flush_lock);
160 ptdev->gpu = gpu;
161
162 dma_set_max_seg_size(ptdev->base.dev, UINT_MAX);
163 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
164 ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits));
165 if (ret)
166 return ret;
167
168 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu");
169 if (irq < 0)
170 return irq;
171
172 ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq,
173 ptdev->iomem + GPU_INT_BASE);
174 if (ret)
175 return ret;
176
177 panthor_gpu_irq_enable_events(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK);
178 panthor_gpu_irq_resume(&ptdev->gpu->irq);
179 return 0;
180 }
181
panthor_gpu_power_changed_on(struct panthor_device * ptdev)182 int panthor_gpu_power_changed_on(struct panthor_device *ptdev)
183 {
184 guard(pm_runtime_active)(ptdev->base.dev);
185
186 panthor_gpu_irq_enable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
187
188 return 0;
189 }
190
panthor_gpu_power_changed_off(struct panthor_device * ptdev)191 void panthor_gpu_power_changed_off(struct panthor_device *ptdev)
192 {
193 guard(pm_runtime_active)(ptdev->base.dev);
194
195 panthor_gpu_irq_disable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
196 }
197
198 /**
199 * panthor_gpu_block_power_off() - Power-off a specific block of the GPU
200 * @ptdev: Device.
201 * @blk_name: Block name.
202 * @pwroff_reg: Power-off register for this block.
203 * @pwrtrans_reg: Power transition register for this block.
204 * @mask: Sub-elements to power-off.
205 * @timeout_us: Timeout in microseconds.
206 *
207 * Return: 0 on success, a negative error code otherwise.
208 */
panthor_gpu_block_power_off(struct panthor_device * ptdev,const char * blk_name,u32 pwroff_reg,u32 pwrtrans_reg,u64 mask,u32 timeout_us)209 int panthor_gpu_block_power_off(struct panthor_device *ptdev,
210 const char *blk_name,
211 u32 pwroff_reg, u32 pwrtrans_reg,
212 u64 mask, u32 timeout_us)
213 {
214 struct panthor_gpu *gpu = ptdev->gpu;
215 u32 val;
216 int ret;
217
218 ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
219 !(mask & val), 100, timeout_us);
220 if (ret) {
221 drm_err(&ptdev->base,
222 "timeout waiting on %s:%llx power transition", blk_name,
223 mask);
224 return ret;
225 }
226
227 gpu_write64(gpu->iomem, pwroff_reg, mask);
228
229 ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
230 !(mask & val), 100, timeout_us);
231 if (ret) {
232 drm_err(&ptdev->base,
233 "timeout waiting on %s:%llx power transition", blk_name,
234 mask);
235 return ret;
236 }
237
238 return 0;
239 }
240
241 /**
242 * panthor_gpu_block_power_on() - Power-on a specific block of the GPU
243 * @ptdev: Device.
244 * @blk_name: Block name.
245 * @pwron_reg: Power-on register for this block.
246 * @pwrtrans_reg: Power transition register for this block.
247 * @rdy_reg: Power transition ready register.
248 * @mask: Sub-elements to power-on.
249 * @timeout_us: Timeout in microseconds.
250 *
251 * Return: 0 on success, a negative error code otherwise.
252 */
panthor_gpu_block_power_on(struct panthor_device * ptdev,const char * blk_name,u32 pwron_reg,u32 pwrtrans_reg,u32 rdy_reg,u64 mask,u32 timeout_us)253 int panthor_gpu_block_power_on(struct panthor_device *ptdev,
254 const char *blk_name,
255 u32 pwron_reg, u32 pwrtrans_reg,
256 u32 rdy_reg, u64 mask, u32 timeout_us)
257 {
258 struct panthor_gpu *gpu = ptdev->gpu;
259 u32 val;
260 int ret;
261
262 ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
263 !(mask & val), 100, timeout_us);
264 if (ret) {
265 drm_err(&ptdev->base,
266 "timeout waiting on %s:%llx power transition", blk_name,
267 mask);
268 return ret;
269 }
270
271 gpu_write64(gpu->iomem, pwron_reg, mask);
272
273 ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, rdy_reg, val,
274 (mask & val) == val,
275 100, timeout_us);
276 if (ret) {
277 drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness",
278 blk_name, mask);
279 return ret;
280 }
281
282 return 0;
283 }
284
panthor_gpu_l2_power_off(struct panthor_device * ptdev)285 void panthor_gpu_l2_power_off(struct panthor_device *ptdev)
286 {
287 panthor_gpu_power_off(ptdev, L2, ptdev->gpu_info.l2_present, 20000);
288 }
289
290 /**
291 * panthor_gpu_l2_power_on() - Power-on the L2-cache
292 * @ptdev: Device.
293 *
294 * Return: 0 on success, a negative error code otherwise.
295 */
panthor_gpu_l2_power_on(struct panthor_device * ptdev)296 int panthor_gpu_l2_power_on(struct panthor_device *ptdev)
297 {
298 if (ptdev->gpu_info.l2_present != 1) {
299 /*
300 * Only support one core group now.
301 * ~(l2_present - 1) unsets all bits in l2_present except
302 * the bottom bit. (l2_present - 2) has all the bits in
303 * the first core group set. AND them together to generate
304 * a mask of cores in the first core group.
305 */
306 u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) &
307 (ptdev->gpu_info.l2_present - 2);
308 drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n",
309 hweight64(core_mask),
310 hweight64(ptdev->gpu_info.shader_present));
311 }
312
313 /* Set the desired coherency mode and L2 config before the power up of L2 */
314 panthor_gpu_coherency_set(ptdev);
315 panthor_gpu_l2_config_set(ptdev);
316
317 return panthor_gpu_power_on(ptdev, L2, 1, 20000);
318 }
319
320 /**
321 * panthor_gpu_flush_caches() - Flush caches
322 * @ptdev: Device.
323 * @l2: L2 flush type.
324 * @lsc: LSC flush type.
325 * @other: Other flush type.
326 *
327 * Return: 0 on success, a negative error code otherwise.
328 */
panthor_gpu_flush_caches(struct panthor_device * ptdev,u32 l2,u32 lsc,u32 other)329 int panthor_gpu_flush_caches(struct panthor_device *ptdev,
330 u32 l2, u32 lsc, u32 other)
331 {
332 struct panthor_gpu *gpu = ptdev->gpu;
333 unsigned long flags;
334 int ret = 0;
335
336 /* Serialize cache flush operations. */
337 guard(mutex)(&ptdev->gpu->cache_flush_lock);
338
339 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
340 if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
341 ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
342 gpu_write(gpu->iomem, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
343 } else {
344 ret = -EIO;
345 }
346 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
347
348 if (ret)
349 return ret;
350
351 if (!wait_event_timeout(ptdev->gpu->reqs_acked,
352 !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED),
353 msecs_to_jiffies(100))) {
354 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
355 if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
356 !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
357 ret = -ETIMEDOUT;
358 else
359 ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
360 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
361 }
362
363 if (ret) {
364 panthor_device_schedule_reset(ptdev);
365 drm_err(&ptdev->base, "Flush caches timeout");
366 }
367
368 return ret;
369 }
370
371 /**
372 * panthor_gpu_soft_reset() - Issue a soft-reset
373 * @ptdev: Device.
374 *
375 * Return: 0 on success, a negative error code otherwise.
376 */
panthor_gpu_soft_reset(struct panthor_device * ptdev)377 int panthor_gpu_soft_reset(struct panthor_device *ptdev)
378 {
379 struct panthor_gpu *gpu = ptdev->gpu;
380 bool timedout = false;
381 unsigned long flags;
382
383 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
384 if (!drm_WARN_ON(&ptdev->base,
385 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) {
386 ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED;
387 gpu_write(gpu->irq.iomem, INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
388 gpu_write(gpu->iomem, GPU_CMD, GPU_SOFT_RESET);
389 }
390 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
391
392 if (!wait_event_timeout(ptdev->gpu->reqs_acked,
393 !(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED),
394 msecs_to_jiffies(100))) {
395 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
396 if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 &&
397 !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
398 timedout = true;
399 else
400 ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED;
401 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
402 }
403
404 if (timedout) {
405 drm_err(&ptdev->base, "Soft reset timeout");
406 return -ETIMEDOUT;
407 }
408
409 ptdev->gpu->pending_reqs = 0;
410 return 0;
411 }
412
413 /**
414 * panthor_gpu_suspend() - Suspend the GPU block.
415 * @ptdev: Device.
416 *
417 * Suspend the GPU irq. This should be called last in the suspend procedure,
418 * after all other blocks have been suspented.
419 */
panthor_gpu_suspend(struct panthor_device * ptdev)420 void panthor_gpu_suspend(struct panthor_device *ptdev)
421 {
422 /* On a fast reset, simply power down the L2. */
423 if (!ptdev->reset.fast)
424 panthor_hw_soft_reset(ptdev);
425 else
426 panthor_hw_l2_power_off(ptdev);
427
428 panthor_gpu_irq_suspend(&ptdev->gpu->irq);
429 }
430
431 /**
432 * panthor_gpu_resume() - Resume the GPU block.
433 * @ptdev: Device.
434 *
435 * Resume the IRQ handler and power-on the L2-cache.
436 * The FW takes care of powering the other blocks.
437 */
panthor_gpu_resume(struct panthor_device * ptdev)438 void panthor_gpu_resume(struct panthor_device *ptdev)
439 {
440 panthor_gpu_irq_resume(&ptdev->gpu->irq);
441 panthor_hw_l2_power_on(ptdev);
442 }
443
panthor_gpu_get_timestamp(struct panthor_device * ptdev)444 u64 panthor_gpu_get_timestamp(struct panthor_device *ptdev)
445 {
446 return gpu_read64_counter(ptdev->gpu->iomem, GPU_TIMESTAMP);
447 }
448
panthor_gpu_get_timestamp_offset(struct panthor_device * ptdev)449 u64 panthor_gpu_get_timestamp_offset(struct panthor_device *ptdev)
450 {
451 return gpu_read64(ptdev->gpu->iomem, GPU_TIMESTAMP_OFFSET);
452 }
453
panthor_gpu_get_cycle_count(struct panthor_device * ptdev)454 u64 panthor_gpu_get_cycle_count(struct panthor_device *ptdev)
455 {
456 return gpu_read64_counter(ptdev->gpu->iomem, GPU_CYCLE_COUNT);
457 }
458
panthor_gpu_coherency_init(struct panthor_device * ptdev)459 int panthor_gpu_coherency_init(struct panthor_device *ptdev)
460 {
461 BUILD_BUG_ON(GPU_COHERENCY_NONE != DRM_PANTHOR_GPU_COHERENCY_NONE);
462 BUILD_BUG_ON(GPU_COHERENCY_ACE_LITE != DRM_PANTHOR_GPU_COHERENCY_ACE_LITE);
463 BUILD_BUG_ON(GPU_COHERENCY_ACE != DRM_PANTHOR_GPU_COHERENCY_ACE);
464
465 /* Start with no coherency, and update it if the device is flagged coherent. */
466 ptdev->gpu_info.selected_coherency = GPU_COHERENCY_NONE;
467 ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
468
469 if (!ptdev->coherent)
470 return 0;
471
472 /* Check if the ACE-Lite coherency protocol is actually supported by the GPU.
473 * ACE protocol has never been supported for command stream frontend GPUs.
474 */
475 if ((gpu_read(ptdev->gpu->iomem, GPU_COHERENCY_FEATURES) &
476 GPU_COHERENCY_PROT_BIT(ACE_LITE))) {
477 ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE;
478 return 0;
479 }
480
481 drm_err(&ptdev->base, "Coherency not supported by the device");
482 return -ENOTSUPP;
483 }
484