1 /* SPDX-License-Identifier: GPL-2.0 or MIT */ 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 4 /* Copyright 2023 Collabora ltd. */ 5 6 #ifndef __PANTHOR_DEVICE_H__ 7 #define __PANTHOR_DEVICE_H__ 8 9 #include <linux/atomic.h> 10 #include <linux/io-pgtable.h> 11 #include <linux/regulator/consumer.h> 12 #include <linux/sched.h> 13 #include <linux/spinlock.h> 14 15 #include <drm/drm_device.h> 16 #include <drm/drm_mm.h> 17 #include <drm/gpu_scheduler.h> 18 #include <drm/panthor_drm.h> 19 20 struct panthor_csf; 21 struct panthor_csf_ctx; 22 struct panthor_device; 23 struct panthor_gpu; 24 struct panthor_group_pool; 25 struct panthor_heap_pool; 26 struct panthor_job; 27 struct panthor_mmu; 28 struct panthor_fw; 29 struct panthor_perfcnt; 30 struct panthor_vm; 31 struct panthor_vm_pool; 32 33 /** 34 * enum panthor_device_pm_state - PM state 35 */ 36 enum panthor_device_pm_state { 37 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */ 38 PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0, 39 40 /** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */ 41 PANTHOR_DEVICE_PM_STATE_RESUMING, 42 43 /** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */ 44 PANTHOR_DEVICE_PM_STATE_ACTIVE, 45 46 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */ 47 PANTHOR_DEVICE_PM_STATE_SUSPENDING, 48 }; 49 50 /** 51 * struct panthor_irq - IRQ data 52 * 53 * Used to automate IRQ handling for the 3 different IRQs we have in this driver. 54 */ 55 struct panthor_irq { 56 /** @ptdev: Panthor device */ 57 struct panthor_device *ptdev; 58 59 /** @irq: IRQ number. */ 60 int irq; 61 62 /** @mask: Current mask being applied to xxx_INT_MASK. */ 63 u32 mask; 64 65 /** @suspended: Set to true when the IRQ is suspended. */ 66 atomic_t suspended; 67 }; 68 69 /** 70 * enum panthor_device_profiling_mode - Profiling state 71 */ 72 enum panthor_device_profiling_flags { 73 /** @PANTHOR_DEVICE_PROFILING_DISABLED: Profiling is disabled. */ 74 PANTHOR_DEVICE_PROFILING_DISABLED = 0, 75 76 /** @PANTHOR_DEVICE_PROFILING_CYCLES: Sampling job cycles. */ 77 PANTHOR_DEVICE_PROFILING_CYCLES = BIT(0), 78 79 /** @PANTHOR_DEVICE_PROFILING_TIMESTAMP: Sampling job timestamp. */ 80 PANTHOR_DEVICE_PROFILING_TIMESTAMP = BIT(1), 81 82 /** @PANTHOR_DEVICE_PROFILING_ALL: Sampling everything. */ 83 PANTHOR_DEVICE_PROFILING_ALL = 84 PANTHOR_DEVICE_PROFILING_CYCLES | 85 PANTHOR_DEVICE_PROFILING_TIMESTAMP, 86 }; 87 88 /** 89 * struct panthor_device - Panthor device 90 */ 91 struct panthor_device { 92 /** @base: Base drm_device. */ 93 struct drm_device base; 94 95 /** @phys_addr: Physical address of the iomem region. */ 96 phys_addr_t phys_addr; 97 98 /** @iomem: CPU mapping of the IOMEM region. */ 99 void __iomem *iomem; 100 101 /** @clks: GPU clocks. */ 102 struct { 103 /** @core: Core clock. */ 104 struct clk *core; 105 106 /** @stacks: Stacks clock. This clock is optional. */ 107 struct clk *stacks; 108 109 /** @coregroup: Core group clock. This clock is optional. */ 110 struct clk *coregroup; 111 } clks; 112 113 /** @coherent: True if the CPU/GPU are memory coherent. */ 114 bool coherent; 115 116 /** @gpu_info: GPU information. */ 117 struct drm_panthor_gpu_info gpu_info; 118 119 /** @csif_info: Command stream interface information. */ 120 struct drm_panthor_csif_info csif_info; 121 122 /** @gpu: GPU management data. */ 123 struct panthor_gpu *gpu; 124 125 /** @fw: FW management data. */ 126 struct panthor_fw *fw; 127 128 /** @mmu: MMU management data. */ 129 struct panthor_mmu *mmu; 130 131 /** @scheduler: Scheduler management data. */ 132 struct panthor_scheduler *scheduler; 133 134 /** @devfreq: Device frequency scaling management data. */ 135 struct panthor_devfreq *devfreq; 136 137 /** @unplug: Device unplug related fields. */ 138 struct { 139 /** @lock: Lock used to serialize unplug operations. */ 140 struct mutex lock; 141 142 /** 143 * @done: Completion object signaled when the unplug 144 * operation is done. 145 */ 146 struct completion done; 147 } unplug; 148 149 /** @reset: Reset related fields. */ 150 struct { 151 /** @wq: Ordered worqueud used to schedule reset operations. */ 152 struct workqueue_struct *wq; 153 154 /** @work: Reset work. */ 155 struct work_struct work; 156 157 /** @pending: Set to true if a reset is pending. */ 158 atomic_t pending; 159 } reset; 160 161 /** @pm: Power management related data. */ 162 struct { 163 /** @state: Power state. */ 164 atomic_t state; 165 166 /** 167 * @mmio_lock: Lock protecting MMIO userspace CPU mappings. 168 * 169 * This is needed to ensure we map the dummy IO pages when 170 * the device is being suspended, and the real IO pages when 171 * the device is being resumed. We can't just do with the 172 * state atomicity to deal with this race. 173 */ 174 struct mutex mmio_lock; 175 176 /** 177 * @dummy_latest_flush: Dummy LATEST_FLUSH page. 178 * 179 * Used to replace the real LATEST_FLUSH page when the GPU 180 * is suspended. 181 */ 182 struct page *dummy_latest_flush; 183 } pm; 184 185 /** @profile_mask: User-set profiling flags for job accounting. */ 186 u32 profile_mask; 187 188 /** @current_frequency: Device clock frequency at present. Set by DVFS*/ 189 unsigned long current_frequency; 190 191 /** @fast_rate: Maximum device clock frequency. Set by DVFS */ 192 unsigned long fast_rate; 193 }; 194 195 struct panthor_gpu_usage { 196 u64 time; 197 u64 cycles; 198 }; 199 200 /** 201 * struct panthor_file - Panthor file 202 */ 203 struct panthor_file { 204 /** @ptdev: Device attached to this file. */ 205 struct panthor_device *ptdev; 206 207 /** @vms: VM pool attached to this file. */ 208 struct panthor_vm_pool *vms; 209 210 /** @groups: Scheduling group pool attached to this file. */ 211 struct panthor_group_pool *groups; 212 213 /** @stats: cycle and timestamp measures for job execution. */ 214 struct panthor_gpu_usage stats; 215 }; 216 217 int panthor_device_init(struct panthor_device *ptdev); 218 void panthor_device_unplug(struct panthor_device *ptdev); 219 220 /** 221 * panthor_device_schedule_reset() - Schedules a reset operation 222 */ 223 static inline void panthor_device_schedule_reset(struct panthor_device *ptdev) 224 { 225 if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) && 226 atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE) 227 queue_work(ptdev->reset.wq, &ptdev->reset.work); 228 } 229 230 /** 231 * panthor_device_reset_is_pending() - Checks if a reset is pending. 232 * 233 * Return: true if a reset is pending, false otherwise. 234 */ 235 static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev) 236 { 237 return atomic_read(&ptdev->reset.pending) != 0; 238 } 239 240 int panthor_device_mmap_io(struct panthor_device *ptdev, 241 struct vm_area_struct *vma); 242 243 int panthor_device_resume(struct device *dev); 244 int panthor_device_suspend(struct device *dev); 245 246 enum drm_panthor_exception_type { 247 DRM_PANTHOR_EXCEPTION_OK = 0x00, 248 DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04, 249 DRM_PANTHOR_EXCEPTION_KABOOM = 0x05, 250 DRM_PANTHOR_EXCEPTION_EUREKA = 0x06, 251 DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08, 252 DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f, 253 DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f, 254 DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40, 255 DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41, 256 DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44, 257 DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48, 258 DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49, 259 DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a, 260 DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b, 261 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50, 262 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51, 263 DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55, 264 DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58, 265 DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59, 266 DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a, 267 DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b, 268 DRM_PANTHOR_EXCEPTION_OOM = 0x60, 269 DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68, 270 DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69, 271 DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80, 272 DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88, 273 DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89, 274 DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a, 275 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0, 276 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1, 277 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2, 278 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3, 279 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4, 280 DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8, 281 DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9, 282 DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca, 283 DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb, 284 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9, 285 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda, 286 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb, 287 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0, 288 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4, 289 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5, 290 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6, 291 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7, 292 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8, 293 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9, 294 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea, 295 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb, 296 }; 297 298 /** 299 * panthor_exception_is_fault() - Checks if an exception is a fault. 300 * 301 * Return: true if the exception is a fault, false otherwise. 302 */ 303 static inline bool 304 panthor_exception_is_fault(u32 exception_code) 305 { 306 return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT; 307 } 308 309 const char *panthor_exception_name(struct panthor_device *ptdev, 310 u32 exception_code); 311 312 /** 313 * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt 314 * registration function. 315 * 316 * The boiler-plate to gracefully deal with shared interrupts is 317 * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER() 318 * just after the actual handler. The handler prototype is: 319 * 320 * void (*handler)(struct panthor_device *, u32 status); 321 */ 322 #define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \ 323 static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \ 324 { \ 325 struct panthor_irq *pirq = data; \ 326 struct panthor_device *ptdev = pirq->ptdev; \ 327 \ 328 if (atomic_read(&pirq->suspended)) \ 329 return IRQ_NONE; \ 330 if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \ 331 return IRQ_NONE; \ 332 \ 333 gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \ 334 return IRQ_WAKE_THREAD; \ 335 } \ 336 \ 337 static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data) \ 338 { \ 339 struct panthor_irq *pirq = data; \ 340 struct panthor_device *ptdev = pirq->ptdev; \ 341 irqreturn_t ret = IRQ_NONE; \ 342 \ 343 while (true) { \ 344 u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \ 345 \ 346 if (!status) \ 347 break; \ 348 \ 349 gpu_write(ptdev, __reg_prefix ## _INT_CLEAR, status); \ 350 \ 351 __handler(ptdev, status); \ 352 ret = IRQ_HANDLED; \ 353 } \ 354 \ 355 if (!atomic_read(&pirq->suspended)) \ 356 gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ 357 \ 358 return ret; \ 359 } \ 360 \ 361 static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \ 362 { \ 363 pirq->mask = 0; \ 364 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ 365 synchronize_irq(pirq->irq); \ 366 atomic_set(&pirq->suspended, true); \ 367 } \ 368 \ 369 static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask) \ 370 { \ 371 atomic_set(&pirq->suspended, false); \ 372 pirq->mask = mask; \ 373 gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \ 374 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask); \ 375 } \ 376 \ 377 static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ 378 struct panthor_irq *pirq, \ 379 int irq, u32 mask) \ 380 { \ 381 pirq->ptdev = ptdev; \ 382 pirq->irq = irq; \ 383 panthor_ ## __name ## _irq_resume(pirq, mask); \ 384 \ 385 return devm_request_threaded_irq(ptdev->base.dev, irq, \ 386 panthor_ ## __name ## _irq_raw_handler, \ 387 panthor_ ## __name ## _irq_threaded_handler, \ 388 IRQF_SHARED, KBUILD_MODNAME "-" # __name, \ 389 pirq); \ 390 } 391 392 extern struct workqueue_struct *panthor_cleanup_wq; 393 394 #endif 395