1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/bitfield.h> 7 #include <linux/bitmap.h> 8 #include <linux/delay.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/iopoll.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include <drm/drm_drv.h> 17 #include <drm/drm_managed.h> 18 19 #include "panthor_device.h" 20 #include "panthor_gpu.h" 21 #include "panthor_regs.h" 22 23 /** 24 * struct panthor_gpu - GPU block management data. 25 */ 26 struct panthor_gpu { 27 /** @irq: GPU irq. */ 28 struct panthor_irq irq; 29 30 /** @reqs_lock: Lock protecting access to pending_reqs. */ 31 spinlock_t reqs_lock; 32 33 /** @pending_reqs: Pending GPU requests. */ 34 u32 pending_reqs; 35 36 /** @reqs_acked: GPU request wait queue. */ 37 wait_queue_head_t reqs_acked; 38 39 /** @cache_flush_lock: Lock to serialize cache flushes */ 40 struct mutex cache_flush_lock; 41 }; 42 43 #define GPU_INTERRUPTS_MASK \ 44 (GPU_IRQ_FAULT | \ 45 GPU_IRQ_PROTM_FAULT | \ 46 GPU_IRQ_RESET_COMPLETED | \ 47 GPU_IRQ_CLEAN_CACHES_COMPLETED) 48 49 static void panthor_gpu_coherency_set(struct panthor_device *ptdev) 50 { 51 gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, 52 ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); 53 } 54 55 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) 56 { 57 const struct panthor_soc_data *data = ptdev->soc_data; 58 u32 l2_config; 59 u32 i; 60 61 if (!data || !data->asn_hash_enable) 62 return; 63 64 if (GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id) < 11) { 65 drm_err(&ptdev->base, "Custom ASN hash not supported by the device"); 66 return; 67 } 68 69 for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++) 70 gpu_write(ptdev, GPU_ASN_HASH(i), data->asn_hash[i]); 71 72 l2_config = gpu_read(ptdev, GPU_L2_CONFIG); 73 l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE; 74 gpu_write(ptdev, GPU_L2_CONFIG, l2_config); 75 } 76 77 static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) 78 { 79 gpu_write(ptdev, GPU_INT_CLEAR, status); 80 81 if (status & GPU_IRQ_FAULT) { 82 u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS); 83 u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR); 84 85 drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n", 86 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF), 87 address); 88 } 89 if (status & GPU_IRQ_PROTM_FAULT) 90 drm_warn(&ptdev->base, "GPU Fault in protected mode\n"); 91 92 spin_lock(&ptdev->gpu->reqs_lock); 93 if (status & ptdev->gpu->pending_reqs) { 94 ptdev->gpu->pending_reqs &= ~status; 95 wake_up_all(&ptdev->gpu->reqs_acked); 96 } 97 spin_unlock(&ptdev->gpu->reqs_lock); 98 } 99 PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler); 100 101 /** 102 * panthor_gpu_unplug() - Called when the GPU is unplugged. 103 * @ptdev: Device to unplug. 104 */ 105 void panthor_gpu_unplug(struct panthor_device *ptdev) 106 { 107 unsigned long flags; 108 109 /* Make sure the IRQ handler is not running after that point. */ 110 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) 111 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 112 113 /* Wake-up all waiters. */ 114 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 115 ptdev->gpu->pending_reqs = 0; 116 wake_up_all(&ptdev->gpu->reqs_acked); 117 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 118 } 119 120 /** 121 * panthor_gpu_init() - Initialize the GPU block 122 * @ptdev: Device. 123 * 124 * Return: 0 on success, a negative error code otherwise. 125 */ 126 int panthor_gpu_init(struct panthor_device *ptdev) 127 { 128 struct panthor_gpu *gpu; 129 u32 pa_bits; 130 int ret, irq; 131 132 gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL); 133 if (!gpu) 134 return -ENOMEM; 135 136 spin_lock_init(&gpu->reqs_lock); 137 init_waitqueue_head(&gpu->reqs_acked); 138 mutex_init(&gpu->cache_flush_lock); 139 ptdev->gpu = gpu; 140 141 dma_set_max_seg_size(ptdev->base.dev, UINT_MAX); 142 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features); 143 ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits)); 144 if (ret) 145 return ret; 146 147 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); 148 if (irq < 0) 149 return irq; 150 151 ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); 152 if (ret) 153 return ret; 154 155 return 0; 156 } 157 158 /** 159 * panthor_gpu_block_power_off() - Power-off a specific block of the GPU 160 * @ptdev: Device. 161 * @blk_name: Block name. 162 * @pwroff_reg: Power-off register for this block. 163 * @pwrtrans_reg: Power transition register for this block. 164 * @mask: Sub-elements to power-off. 165 * @timeout_us: Timeout in microseconds. 166 * 167 * Return: 0 on success, a negative error code otherwise. 168 */ 169 int panthor_gpu_block_power_off(struct panthor_device *ptdev, 170 const char *blk_name, 171 u32 pwroff_reg, u32 pwrtrans_reg, 172 u64 mask, u32 timeout_us) 173 { 174 u32 val; 175 int ret; 176 177 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 178 !(mask & val), 100, timeout_us); 179 if (ret) { 180 drm_err(&ptdev->base, 181 "timeout waiting on %s:%llx power transition", blk_name, 182 mask); 183 return ret; 184 } 185 186 gpu_write64(ptdev, pwroff_reg, mask); 187 188 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 189 !(mask & val), 100, timeout_us); 190 if (ret) { 191 drm_err(&ptdev->base, 192 "timeout waiting on %s:%llx power transition", blk_name, 193 mask); 194 return ret; 195 } 196 197 return 0; 198 } 199 200 /** 201 * panthor_gpu_block_power_on() - Power-on a specific block of the GPU 202 * @ptdev: Device. 203 * @blk_name: Block name. 204 * @pwron_reg: Power-on register for this block. 205 * @pwrtrans_reg: Power transition register for this block. 206 * @rdy_reg: Power transition ready register. 207 * @mask: Sub-elements to power-on. 208 * @timeout_us: Timeout in microseconds. 209 * 210 * Return: 0 on success, a negative error code otherwise. 211 */ 212 int panthor_gpu_block_power_on(struct panthor_device *ptdev, 213 const char *blk_name, 214 u32 pwron_reg, u32 pwrtrans_reg, 215 u32 rdy_reg, u64 mask, u32 timeout_us) 216 { 217 u32 val; 218 int ret; 219 220 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 221 !(mask & val), 100, timeout_us); 222 if (ret) { 223 drm_err(&ptdev->base, 224 "timeout waiting on %s:%llx power transition", blk_name, 225 mask); 226 return ret; 227 } 228 229 gpu_write64(ptdev, pwron_reg, mask); 230 231 ret = gpu_read64_relaxed_poll_timeout(ptdev, rdy_reg, val, 232 (mask & val) == val, 233 100, timeout_us); 234 if (ret) { 235 drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness", 236 blk_name, mask); 237 return ret; 238 } 239 240 return 0; 241 } 242 243 /** 244 * panthor_gpu_l2_power_on() - Power-on the L2-cache 245 * @ptdev: Device. 246 * 247 * Return: 0 on success, a negative error code otherwise. 248 */ 249 int panthor_gpu_l2_power_on(struct panthor_device *ptdev) 250 { 251 if (ptdev->gpu_info.l2_present != 1) { 252 /* 253 * Only support one core group now. 254 * ~(l2_present - 1) unsets all bits in l2_present except 255 * the bottom bit. (l2_present - 2) has all the bits in 256 * the first core group set. AND them together to generate 257 * a mask of cores in the first core group. 258 */ 259 u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) & 260 (ptdev->gpu_info.l2_present - 2); 261 drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n", 262 hweight64(core_mask), 263 hweight64(ptdev->gpu_info.shader_present)); 264 } 265 266 /* Set the desired coherency mode and L2 config before the power up of L2 */ 267 panthor_gpu_coherency_set(ptdev); 268 panthor_gpu_l2_config_set(ptdev); 269 270 return panthor_gpu_power_on(ptdev, L2, 1, 20000); 271 } 272 273 /** 274 * panthor_gpu_flush_caches() - Flush caches 275 * @ptdev: Device. 276 * @l2: L2 flush type. 277 * @lsc: LSC flush type. 278 * @other: Other flush type. 279 * 280 * Return: 0 on success, a negative error code otherwise. 281 */ 282 int panthor_gpu_flush_caches(struct panthor_device *ptdev, 283 u32 l2, u32 lsc, u32 other) 284 { 285 bool timedout = false; 286 unsigned long flags; 287 288 /* Serialize cache flush operations. */ 289 guard(mutex)(&ptdev->gpu->cache_flush_lock); 290 291 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 292 if (!drm_WARN_ON(&ptdev->base, 293 ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { 294 ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; 295 gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); 296 } 297 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 298 299 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 300 !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), 301 msecs_to_jiffies(100))) { 302 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 303 if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && 304 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) 305 timedout = true; 306 else 307 ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; 308 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 309 } 310 311 if (timedout) { 312 drm_err(&ptdev->base, "Flush caches timeout"); 313 return -ETIMEDOUT; 314 } 315 316 return 0; 317 } 318 319 /** 320 * panthor_gpu_soft_reset() - Issue a soft-reset 321 * @ptdev: Device. 322 * 323 * Return: 0 on success, a negative error code otherwise. 324 */ 325 int panthor_gpu_soft_reset(struct panthor_device *ptdev) 326 { 327 bool timedout = false; 328 unsigned long flags; 329 330 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 331 if (!drm_WARN_ON(&ptdev->base, 332 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) { 333 ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED; 334 gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); 335 gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET); 336 } 337 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 338 339 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 340 !(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED), 341 msecs_to_jiffies(100))) { 342 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 343 if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 && 344 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) 345 timedout = true; 346 else 347 ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED; 348 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 349 } 350 351 if (timedout) { 352 drm_err(&ptdev->base, "Soft reset timeout"); 353 return -ETIMEDOUT; 354 } 355 356 return 0; 357 } 358 359 /** 360 * panthor_gpu_suspend() - Suspend the GPU block. 361 * @ptdev: Device. 362 * 363 * Suspend the GPU irq. This should be called last in the suspend procedure, 364 * after all other blocks have been suspented. 365 */ 366 void panthor_gpu_suspend(struct panthor_device *ptdev) 367 { 368 /* On a fast reset, simply power down the L2. */ 369 if (!ptdev->reset.fast) 370 panthor_gpu_soft_reset(ptdev); 371 else 372 panthor_gpu_power_off(ptdev, L2, 1, 20000); 373 374 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 375 } 376 377 /** 378 * panthor_gpu_resume() - Resume the GPU block. 379 * @ptdev: Device. 380 * 381 * Resume the IRQ handler and power-on the L2-cache. 382 * The FW takes care of powering the other blocks. 383 */ 384 void panthor_gpu_resume(struct panthor_device *ptdev) 385 { 386 panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK); 387 panthor_gpu_l2_power_on(ptdev); 388 } 389 390