1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/bitfield.h> 7 #include <linux/bitmap.h> 8 #include <linux/delay.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/iopoll.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include <drm/drm_drv.h> 17 #include <drm/drm_managed.h> 18 #include <drm/drm_print.h> 19 20 #include "panthor_device.h" 21 #include "panthor_gpu.h" 22 #include "panthor_regs.h" 23 24 /** 25 * struct panthor_gpu - GPU block management data. 26 */ 27 struct panthor_gpu { 28 /** @irq: GPU irq. */ 29 struct panthor_irq irq; 30 31 /** @reqs_lock: Lock protecting access to pending_reqs. */ 32 spinlock_t reqs_lock; 33 34 /** @pending_reqs: Pending GPU requests. */ 35 u32 pending_reqs; 36 37 /** @reqs_acked: GPU request wait queue. */ 38 wait_queue_head_t reqs_acked; 39 40 /** @cache_flush_lock: Lock to serialize cache flushes */ 41 struct mutex cache_flush_lock; 42 }; 43 44 #define GPU_INTERRUPTS_MASK \ 45 (GPU_IRQ_FAULT | \ 46 GPU_IRQ_PROTM_FAULT | \ 47 GPU_IRQ_RESET_COMPLETED | \ 48 GPU_IRQ_CLEAN_CACHES_COMPLETED) 49 50 static void panthor_gpu_coherency_set(struct panthor_device *ptdev) 51 { 52 gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, 53 ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); 54 } 55 56 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) 57 { 58 const struct panthor_soc_data *data = ptdev->soc_data; 59 u32 l2_config; 60 u32 i; 61 62 if (!data || !data->asn_hash_enable) 63 return; 64 65 if (GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id) < 11) { 66 drm_err(&ptdev->base, "Custom ASN hash not supported by the device"); 67 return; 68 } 69 70 for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++) 71 gpu_write(ptdev, GPU_ASN_HASH(i), data->asn_hash[i]); 72 73 l2_config = gpu_read(ptdev, GPU_L2_CONFIG); 74 l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE; 75 gpu_write(ptdev, GPU_L2_CONFIG, l2_config); 76 } 77 78 static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) 79 { 80 gpu_write(ptdev, GPU_INT_CLEAR, status); 81 82 if (status & GPU_IRQ_FAULT) { 83 u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS); 84 u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR); 85 86 drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n", 87 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF), 88 address); 89 } 90 if (status & GPU_IRQ_PROTM_FAULT) 91 drm_warn(&ptdev->base, "GPU Fault in protected mode\n"); 92 93 spin_lock(&ptdev->gpu->reqs_lock); 94 if (status & ptdev->gpu->pending_reqs) { 95 ptdev->gpu->pending_reqs &= ~status; 96 wake_up_all(&ptdev->gpu->reqs_acked); 97 } 98 spin_unlock(&ptdev->gpu->reqs_lock); 99 } 100 PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler); 101 102 /** 103 * panthor_gpu_unplug() - Called when the GPU is unplugged. 104 * @ptdev: Device to unplug. 105 */ 106 void panthor_gpu_unplug(struct panthor_device *ptdev) 107 { 108 unsigned long flags; 109 110 /* Make sure the IRQ handler is not running after that point. */ 111 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) 112 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 113 114 /* Wake-up all waiters. */ 115 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 116 ptdev->gpu->pending_reqs = 0; 117 wake_up_all(&ptdev->gpu->reqs_acked); 118 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 119 } 120 121 /** 122 * panthor_gpu_init() - Initialize the GPU block 123 * @ptdev: Device. 124 * 125 * Return: 0 on success, a negative error code otherwise. 126 */ 127 int panthor_gpu_init(struct panthor_device *ptdev) 128 { 129 struct panthor_gpu *gpu; 130 u32 pa_bits; 131 int ret, irq; 132 133 gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL); 134 if (!gpu) 135 return -ENOMEM; 136 137 spin_lock_init(&gpu->reqs_lock); 138 init_waitqueue_head(&gpu->reqs_acked); 139 mutex_init(&gpu->cache_flush_lock); 140 ptdev->gpu = gpu; 141 142 dma_set_max_seg_size(ptdev->base.dev, UINT_MAX); 143 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features); 144 ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits)); 145 if (ret) 146 return ret; 147 148 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); 149 if (irq < 0) 150 return irq; 151 152 ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); 153 if (ret) 154 return ret; 155 156 return 0; 157 } 158 159 /** 160 * panthor_gpu_block_power_off() - Power-off a specific block of the GPU 161 * @ptdev: Device. 162 * @blk_name: Block name. 163 * @pwroff_reg: Power-off register for this block. 164 * @pwrtrans_reg: Power transition register for this block. 165 * @mask: Sub-elements to power-off. 166 * @timeout_us: Timeout in microseconds. 167 * 168 * Return: 0 on success, a negative error code otherwise. 169 */ 170 int panthor_gpu_block_power_off(struct panthor_device *ptdev, 171 const char *blk_name, 172 u32 pwroff_reg, u32 pwrtrans_reg, 173 u64 mask, u32 timeout_us) 174 { 175 u32 val; 176 int ret; 177 178 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 179 !(mask & val), 100, timeout_us); 180 if (ret) { 181 drm_err(&ptdev->base, 182 "timeout waiting on %s:%llx power transition", blk_name, 183 mask); 184 return ret; 185 } 186 187 gpu_write64(ptdev, pwroff_reg, mask); 188 189 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 190 !(mask & val), 100, timeout_us); 191 if (ret) { 192 drm_err(&ptdev->base, 193 "timeout waiting on %s:%llx power transition", blk_name, 194 mask); 195 return ret; 196 } 197 198 return 0; 199 } 200 201 /** 202 * panthor_gpu_block_power_on() - Power-on a specific block of the GPU 203 * @ptdev: Device. 204 * @blk_name: Block name. 205 * @pwron_reg: Power-on register for this block. 206 * @pwrtrans_reg: Power transition register for this block. 207 * @rdy_reg: Power transition ready register. 208 * @mask: Sub-elements to power-on. 209 * @timeout_us: Timeout in microseconds. 210 * 211 * Return: 0 on success, a negative error code otherwise. 212 */ 213 int panthor_gpu_block_power_on(struct panthor_device *ptdev, 214 const char *blk_name, 215 u32 pwron_reg, u32 pwrtrans_reg, 216 u32 rdy_reg, u64 mask, u32 timeout_us) 217 { 218 u32 val; 219 int ret; 220 221 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 222 !(mask & val), 100, timeout_us); 223 if (ret) { 224 drm_err(&ptdev->base, 225 "timeout waiting on %s:%llx power transition", blk_name, 226 mask); 227 return ret; 228 } 229 230 gpu_write64(ptdev, pwron_reg, mask); 231 232 ret = gpu_read64_relaxed_poll_timeout(ptdev, rdy_reg, val, 233 (mask & val) == val, 234 100, timeout_us); 235 if (ret) { 236 drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness", 237 blk_name, mask); 238 return ret; 239 } 240 241 return 0; 242 } 243 244 /** 245 * panthor_gpu_l2_power_on() - Power-on the L2-cache 246 * @ptdev: Device. 247 * 248 * Return: 0 on success, a negative error code otherwise. 249 */ 250 int panthor_gpu_l2_power_on(struct panthor_device *ptdev) 251 { 252 if (ptdev->gpu_info.l2_present != 1) { 253 /* 254 * Only support one core group now. 255 * ~(l2_present - 1) unsets all bits in l2_present except 256 * the bottom bit. (l2_present - 2) has all the bits in 257 * the first core group set. AND them together to generate 258 * a mask of cores in the first core group. 259 */ 260 u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) & 261 (ptdev->gpu_info.l2_present - 2); 262 drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n", 263 hweight64(core_mask), 264 hweight64(ptdev->gpu_info.shader_present)); 265 } 266 267 /* Set the desired coherency mode and L2 config before the power up of L2 */ 268 panthor_gpu_coherency_set(ptdev); 269 panthor_gpu_l2_config_set(ptdev); 270 271 return panthor_gpu_power_on(ptdev, L2, 1, 20000); 272 } 273 274 /** 275 * panthor_gpu_flush_caches() - Flush caches 276 * @ptdev: Device. 277 * @l2: L2 flush type. 278 * @lsc: LSC flush type. 279 * @other: Other flush type. 280 * 281 * Return: 0 on success, a negative error code otherwise. 282 */ 283 int panthor_gpu_flush_caches(struct panthor_device *ptdev, 284 u32 l2, u32 lsc, u32 other) 285 { 286 bool timedout = false; 287 unsigned long flags; 288 289 /* Serialize cache flush operations. */ 290 guard(mutex)(&ptdev->gpu->cache_flush_lock); 291 292 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 293 if (!drm_WARN_ON(&ptdev->base, 294 ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { 295 ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; 296 gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); 297 } 298 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 299 300 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 301 !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), 302 msecs_to_jiffies(100))) { 303 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 304 if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && 305 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) 306 timedout = true; 307 else 308 ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; 309 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 310 } 311 312 if (timedout) { 313 drm_err(&ptdev->base, "Flush caches timeout"); 314 return -ETIMEDOUT; 315 } 316 317 return 0; 318 } 319 320 /** 321 * panthor_gpu_soft_reset() - Issue a soft-reset 322 * @ptdev: Device. 323 * 324 * Return: 0 on success, a negative error code otherwise. 325 */ 326 int panthor_gpu_soft_reset(struct panthor_device *ptdev) 327 { 328 bool timedout = false; 329 unsigned long flags; 330 331 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 332 if (!drm_WARN_ON(&ptdev->base, 333 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) { 334 ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED; 335 gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); 336 gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET); 337 } 338 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 339 340 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 341 !(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED), 342 msecs_to_jiffies(100))) { 343 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 344 if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 && 345 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) 346 timedout = true; 347 else 348 ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED; 349 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 350 } 351 352 if (timedout) { 353 drm_err(&ptdev->base, "Soft reset timeout"); 354 return -ETIMEDOUT; 355 } 356 357 return 0; 358 } 359 360 /** 361 * panthor_gpu_suspend() - Suspend the GPU block. 362 * @ptdev: Device. 363 * 364 * Suspend the GPU irq. This should be called last in the suspend procedure, 365 * after all other blocks have been suspented. 366 */ 367 void panthor_gpu_suspend(struct panthor_device *ptdev) 368 { 369 /* On a fast reset, simply power down the L2. */ 370 if (!ptdev->reset.fast) 371 panthor_gpu_soft_reset(ptdev); 372 else 373 panthor_gpu_power_off(ptdev, L2, 1, 20000); 374 375 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 376 } 377 378 /** 379 * panthor_gpu_resume() - Resume the GPU block. 380 * @ptdev: Device. 381 * 382 * Resume the IRQ handler and power-on the L2-cache. 383 * The FW takes care of powering the other blocks. 384 */ 385 void panthor_gpu_resume(struct panthor_device *ptdev) 386 { 387 panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK); 388 panthor_gpu_l2_power_on(ptdev); 389 } 390 391