1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/bitfield.h> 7 #include <linux/bitmap.h> 8 #include <linux/delay.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/iopoll.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 16 #include <drm/drm_drv.h> 17 #include <drm/drm_managed.h> 18 #include <drm/drm_print.h> 19 20 #include "panthor_device.h" 21 #include "panthor_gpu.h" 22 #include "panthor_hw.h" 23 #include "panthor_regs.h" 24 25 /** 26 * struct panthor_gpu - GPU block management data. 27 */ 28 struct panthor_gpu { 29 /** @irq: GPU irq. */ 30 struct panthor_irq irq; 31 32 /** @reqs_lock: Lock protecting access to pending_reqs. */ 33 spinlock_t reqs_lock; 34 35 /** @pending_reqs: Pending GPU requests. */ 36 u32 pending_reqs; 37 38 /** @reqs_acked: GPU request wait queue. */ 39 wait_queue_head_t reqs_acked; 40 41 /** @cache_flush_lock: Lock to serialize cache flushes */ 42 struct mutex cache_flush_lock; 43 }; 44 45 #define GPU_INTERRUPTS_MASK \ 46 (GPU_IRQ_FAULT | \ 47 GPU_IRQ_PROTM_FAULT | \ 48 GPU_IRQ_RESET_COMPLETED | \ 49 GPU_IRQ_CLEAN_CACHES_COMPLETED) 50 51 static void panthor_gpu_coherency_set(struct panthor_device *ptdev) 52 { 53 gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, 54 ptdev->gpu_info.selected_coherency); 55 } 56 57 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) 58 { 59 const struct panthor_soc_data *data = ptdev->soc_data; 60 u32 l2_config; 61 u32 i; 62 63 if (!data || !data->asn_hash_enable) 64 return; 65 66 if (GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id) < 11) { 67 drm_err(&ptdev->base, "Custom ASN hash not supported by the device"); 68 return; 69 } 70 71 for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++) 72 gpu_write(ptdev, GPU_ASN_HASH(i), data->asn_hash[i]); 73 74 l2_config = gpu_read(ptdev, GPU_L2_CONFIG); 75 l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE; 76 gpu_write(ptdev, GPU_L2_CONFIG, l2_config); 77 } 78 79 static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) 80 { 81 gpu_write(ptdev, GPU_INT_CLEAR, status); 82 83 if (status & GPU_IRQ_FAULT) { 84 u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS); 85 u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR); 86 87 drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n", 88 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF), 89 address); 90 } 91 if (status & GPU_IRQ_PROTM_FAULT) 92 drm_warn(&ptdev->base, "GPU Fault in protected mode\n"); 93 94 spin_lock(&ptdev->gpu->reqs_lock); 95 if (status & ptdev->gpu->pending_reqs) { 96 ptdev->gpu->pending_reqs &= ~status; 97 wake_up_all(&ptdev->gpu->reqs_acked); 98 } 99 spin_unlock(&ptdev->gpu->reqs_lock); 100 } 101 PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler); 102 103 /** 104 * panthor_gpu_unplug() - Called when the GPU is unplugged. 105 * @ptdev: Device to unplug. 106 */ 107 void panthor_gpu_unplug(struct panthor_device *ptdev) 108 { 109 unsigned long flags; 110 111 /* Make sure the IRQ handler is not running after that point. */ 112 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) 113 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 114 115 /* Wake-up all waiters. */ 116 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 117 ptdev->gpu->pending_reqs = 0; 118 wake_up_all(&ptdev->gpu->reqs_acked); 119 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 120 } 121 122 /** 123 * panthor_gpu_init() - Initialize the GPU block 124 * @ptdev: Device. 125 * 126 * Return: 0 on success, a negative error code otherwise. 127 */ 128 int panthor_gpu_init(struct panthor_device *ptdev) 129 { 130 struct panthor_gpu *gpu; 131 u32 pa_bits; 132 int ret, irq; 133 134 gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL); 135 if (!gpu) 136 return -ENOMEM; 137 138 spin_lock_init(&gpu->reqs_lock); 139 init_waitqueue_head(&gpu->reqs_acked); 140 mutex_init(&gpu->cache_flush_lock); 141 ptdev->gpu = gpu; 142 143 dma_set_max_seg_size(ptdev->base.dev, UINT_MAX); 144 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features); 145 ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits)); 146 if (ret) 147 return ret; 148 149 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); 150 if (irq < 0) 151 return irq; 152 153 ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); 154 if (ret) 155 return ret; 156 157 return 0; 158 } 159 160 /** 161 * panthor_gpu_block_power_off() - Power-off a specific block of the GPU 162 * @ptdev: Device. 163 * @blk_name: Block name. 164 * @pwroff_reg: Power-off register for this block. 165 * @pwrtrans_reg: Power transition register for this block. 166 * @mask: Sub-elements to power-off. 167 * @timeout_us: Timeout in microseconds. 168 * 169 * Return: 0 on success, a negative error code otherwise. 170 */ 171 int panthor_gpu_block_power_off(struct panthor_device *ptdev, 172 const char *blk_name, 173 u32 pwroff_reg, u32 pwrtrans_reg, 174 u64 mask, u32 timeout_us) 175 { 176 u32 val; 177 int ret; 178 179 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 180 !(mask & val), 100, timeout_us); 181 if (ret) { 182 drm_err(&ptdev->base, 183 "timeout waiting on %s:%llx power transition", blk_name, 184 mask); 185 return ret; 186 } 187 188 gpu_write64(ptdev, pwroff_reg, mask); 189 190 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 191 !(mask & val), 100, timeout_us); 192 if (ret) { 193 drm_err(&ptdev->base, 194 "timeout waiting on %s:%llx power transition", blk_name, 195 mask); 196 return ret; 197 } 198 199 return 0; 200 } 201 202 /** 203 * panthor_gpu_block_power_on() - Power-on a specific block of the GPU 204 * @ptdev: Device. 205 * @blk_name: Block name. 206 * @pwron_reg: Power-on register for this block. 207 * @pwrtrans_reg: Power transition register for this block. 208 * @rdy_reg: Power transition ready register. 209 * @mask: Sub-elements to power-on. 210 * @timeout_us: Timeout in microseconds. 211 * 212 * Return: 0 on success, a negative error code otherwise. 213 */ 214 int panthor_gpu_block_power_on(struct panthor_device *ptdev, 215 const char *blk_name, 216 u32 pwron_reg, u32 pwrtrans_reg, 217 u32 rdy_reg, u64 mask, u32 timeout_us) 218 { 219 u32 val; 220 int ret; 221 222 ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val, 223 !(mask & val), 100, timeout_us); 224 if (ret) { 225 drm_err(&ptdev->base, 226 "timeout waiting on %s:%llx power transition", blk_name, 227 mask); 228 return ret; 229 } 230 231 gpu_write64(ptdev, pwron_reg, mask); 232 233 ret = gpu_read64_relaxed_poll_timeout(ptdev, rdy_reg, val, 234 (mask & val) == val, 235 100, timeout_us); 236 if (ret) { 237 drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness", 238 blk_name, mask); 239 return ret; 240 } 241 242 return 0; 243 } 244 245 void panthor_gpu_l2_power_off(struct panthor_device *ptdev) 246 { 247 panthor_gpu_power_off(ptdev, L2, ptdev->gpu_info.l2_present, 20000); 248 } 249 250 /** 251 * panthor_gpu_l2_power_on() - Power-on the L2-cache 252 * @ptdev: Device. 253 * 254 * Return: 0 on success, a negative error code otherwise. 255 */ 256 int panthor_gpu_l2_power_on(struct panthor_device *ptdev) 257 { 258 if (ptdev->gpu_info.l2_present != 1) { 259 /* 260 * Only support one core group now. 261 * ~(l2_present - 1) unsets all bits in l2_present except 262 * the bottom bit. (l2_present - 2) has all the bits in 263 * the first core group set. AND them together to generate 264 * a mask of cores in the first core group. 265 */ 266 u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) & 267 (ptdev->gpu_info.l2_present - 2); 268 drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n", 269 hweight64(core_mask), 270 hweight64(ptdev->gpu_info.shader_present)); 271 } 272 273 /* Set the desired coherency mode and L2 config before the power up of L2 */ 274 panthor_gpu_coherency_set(ptdev); 275 panthor_gpu_l2_config_set(ptdev); 276 277 return panthor_gpu_power_on(ptdev, L2, 1, 20000); 278 } 279 280 /** 281 * panthor_gpu_flush_caches() - Flush caches 282 * @ptdev: Device. 283 * @l2: L2 flush type. 284 * @lsc: LSC flush type. 285 * @other: Other flush type. 286 * 287 * Return: 0 on success, a negative error code otherwise. 288 */ 289 int panthor_gpu_flush_caches(struct panthor_device *ptdev, 290 u32 l2, u32 lsc, u32 other) 291 { 292 unsigned long flags; 293 int ret = 0; 294 295 /* Serialize cache flush operations. */ 296 guard(mutex)(&ptdev->gpu->cache_flush_lock); 297 298 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 299 if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { 300 ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; 301 gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); 302 } else { 303 ret = -EIO; 304 } 305 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 306 307 if (ret) 308 return ret; 309 310 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 311 !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), 312 msecs_to_jiffies(100))) { 313 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 314 if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && 315 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) 316 ret = -ETIMEDOUT; 317 else 318 ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; 319 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 320 } 321 322 if (ret) { 323 panthor_device_schedule_reset(ptdev); 324 drm_err(&ptdev->base, "Flush caches timeout"); 325 } 326 327 return ret; 328 } 329 330 /** 331 * panthor_gpu_soft_reset() - Issue a soft-reset 332 * @ptdev: Device. 333 * 334 * Return: 0 on success, a negative error code otherwise. 335 */ 336 int panthor_gpu_soft_reset(struct panthor_device *ptdev) 337 { 338 bool timedout = false; 339 unsigned long flags; 340 341 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 342 if (!drm_WARN_ON(&ptdev->base, 343 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) { 344 ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED; 345 gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); 346 gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET); 347 } 348 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 349 350 if (!wait_event_timeout(ptdev->gpu->reqs_acked, 351 !(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED), 352 msecs_to_jiffies(100))) { 353 spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); 354 if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 && 355 !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) 356 timedout = true; 357 else 358 ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED; 359 spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); 360 } 361 362 if (timedout) { 363 drm_err(&ptdev->base, "Soft reset timeout"); 364 return -ETIMEDOUT; 365 } 366 367 ptdev->gpu->pending_reqs = 0; 368 return 0; 369 } 370 371 /** 372 * panthor_gpu_suspend() - Suspend the GPU block. 373 * @ptdev: Device. 374 * 375 * Suspend the GPU irq. This should be called last in the suspend procedure, 376 * after all other blocks have been suspented. 377 */ 378 void panthor_gpu_suspend(struct panthor_device *ptdev) 379 { 380 /* On a fast reset, simply power down the L2. */ 381 if (!ptdev->reset.fast) 382 panthor_hw_soft_reset(ptdev); 383 else 384 panthor_hw_l2_power_off(ptdev); 385 386 panthor_gpu_irq_suspend(&ptdev->gpu->irq); 387 } 388 389 /** 390 * panthor_gpu_resume() - Resume the GPU block. 391 * @ptdev: Device. 392 * 393 * Resume the IRQ handler and power-on the L2-cache. 394 * The FW takes care of powering the other blocks. 395 */ 396 void panthor_gpu_resume(struct panthor_device *ptdev) 397 { 398 panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK); 399 panthor_hw_l2_power_on(ptdev); 400 } 401 402