1 /* SPDX-License-Identifier: GPL-2.0 or MIT */ 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 4 /* Copyright 2023 Collabora ltd. */ 5 6 #ifndef __PANTHOR_DEVICE_H__ 7 #define __PANTHOR_DEVICE_H__ 8 9 #include <linux/atomic.h> 10 #include <linux/io-pgtable.h> 11 #include <linux/regulator/consumer.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/sched.h> 14 #include <linux/spinlock.h> 15 16 #include <drm/drm_device.h> 17 #include <drm/drm_mm.h> 18 #include <drm/gpu_scheduler.h> 19 #include <drm/panthor_drm.h> 20 21 struct panthor_csf; 22 struct panthor_csf_ctx; 23 struct panthor_device; 24 struct panthor_gpu; 25 struct panthor_group_pool; 26 struct panthor_heap_pool; 27 struct panthor_hw; 28 struct panthor_job; 29 struct panthor_mmu; 30 struct panthor_fw; 31 struct panthor_perfcnt; 32 struct panthor_pwr; 33 struct panthor_vm; 34 struct panthor_vm_pool; 35 36 /** 37 * struct panthor_soc_data - Panthor SoC Data 38 */ 39 struct panthor_soc_data { 40 /** @asn_hash_enable: True if GPU_L2_CONFIG_ASN_HASH_ENABLE must be set. */ 41 bool asn_hash_enable; 42 43 /** @asn_hash: ASN_HASH values when asn_hash_enable is true. */ 44 u32 asn_hash[3]; 45 }; 46 47 /** 48 * enum panthor_device_pm_state - PM state 49 */ 50 enum panthor_device_pm_state { 51 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */ 52 PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0, 53 54 /** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */ 55 PANTHOR_DEVICE_PM_STATE_RESUMING, 56 57 /** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */ 58 PANTHOR_DEVICE_PM_STATE_ACTIVE, 59 60 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */ 61 PANTHOR_DEVICE_PM_STATE_SUSPENDING, 62 }; 63 64 enum panthor_irq_state { 65 /** @PANTHOR_IRQ_STATE_ACTIVE: IRQ is active and ready to process events. */ 66 PANTHOR_IRQ_STATE_ACTIVE = 0, 67 /** @PANTHOR_IRQ_STATE_PROCESSING: IRQ is currently processing events. */ 68 PANTHOR_IRQ_STATE_PROCESSING, 69 /** @PANTHOR_IRQ_STATE_SUSPENDED: IRQ is suspended. */ 70 PANTHOR_IRQ_STATE_SUSPENDED, 71 /** @PANTHOR_IRQ_STATE_SUSPENDING: IRQ is being suspended. */ 72 PANTHOR_IRQ_STATE_SUSPENDING, 73 }; 74 75 /** 76 * struct panthor_irq - IRQ data 77 * 78 * Used to automate IRQ handling for the 3 different IRQs we have in this driver. 79 */ 80 struct panthor_irq { 81 /** @ptdev: Panthor device */ 82 struct panthor_device *ptdev; 83 84 /** @irq: IRQ number. */ 85 int irq; 86 87 /** @mask: Values to write to xxx_INT_MASK if active. */ 88 u32 mask; 89 90 /** 91 * @mask_lock: protects modifications to _INT_MASK and @mask. 92 * 93 * In paths where _INT_MASK is updated based on a state 94 * transition/check, it's crucial for the state update/check to be 95 * inside the locked section, otherwise it introduces a race window 96 * leading to potential _INT_MASK inconsistencies. 97 */ 98 spinlock_t mask_lock; 99 100 /** @state: one of &enum panthor_irq_state reflecting the current state. */ 101 atomic_t state; 102 }; 103 104 /** 105 * enum panthor_device_profiling_mode - Profiling state 106 */ 107 enum panthor_device_profiling_flags { 108 /** @PANTHOR_DEVICE_PROFILING_DISABLED: Profiling is disabled. */ 109 PANTHOR_DEVICE_PROFILING_DISABLED = 0, 110 111 /** @PANTHOR_DEVICE_PROFILING_CYCLES: Sampling job cycles. */ 112 PANTHOR_DEVICE_PROFILING_CYCLES = BIT(0), 113 114 /** @PANTHOR_DEVICE_PROFILING_TIMESTAMP: Sampling job timestamp. */ 115 PANTHOR_DEVICE_PROFILING_TIMESTAMP = BIT(1), 116 117 /** @PANTHOR_DEVICE_PROFILING_ALL: Sampling everything. */ 118 PANTHOR_DEVICE_PROFILING_ALL = 119 PANTHOR_DEVICE_PROFILING_CYCLES | 120 PANTHOR_DEVICE_PROFILING_TIMESTAMP, 121 }; 122 123 /** 124 * struct panthor_device - Panthor device 125 */ 126 struct panthor_device { 127 /** @base: Base drm_device. */ 128 struct drm_device base; 129 130 /** @soc_data: Optional SoC data. */ 131 const struct panthor_soc_data *soc_data; 132 133 /** @phys_addr: Physical address of the iomem region. */ 134 phys_addr_t phys_addr; 135 136 /** @iomem: CPU mapping of the IOMEM region. */ 137 void __iomem *iomem; 138 139 /** @clks: GPU clocks. */ 140 struct { 141 /** @core: Core clock. */ 142 struct clk *core; 143 144 /** @stacks: Stacks clock. This clock is optional. */ 145 struct clk *stacks; 146 147 /** @coregroup: Core group clock. This clock is optional. */ 148 struct clk *coregroup; 149 } clks; 150 151 /** @coherent: True if the CPU/GPU are memory coherent. */ 152 bool coherent; 153 154 /** @gpu_info: GPU information. */ 155 struct drm_panthor_gpu_info gpu_info; 156 157 /** @csif_info: Command stream interface information. */ 158 struct drm_panthor_csif_info csif_info; 159 160 /** @hw: GPU-specific data. */ 161 struct panthor_hw *hw; 162 163 /** @pwr: Power control management data. */ 164 struct panthor_pwr *pwr; 165 166 /** @gpu: GPU management data. */ 167 struct panthor_gpu *gpu; 168 169 /** @fw: FW management data. */ 170 struct panthor_fw *fw; 171 172 /** @mmu: MMU management data. */ 173 struct panthor_mmu *mmu; 174 175 /** @scheduler: Scheduler management data. */ 176 struct panthor_scheduler *scheduler; 177 178 /** @devfreq: Device frequency scaling management data. */ 179 struct panthor_devfreq *devfreq; 180 181 /** @unplug: Device unplug related fields. */ 182 struct { 183 /** @lock: Lock used to serialize unplug operations. */ 184 struct mutex lock; 185 186 /** 187 * @done: Completion object signaled when the unplug 188 * operation is done. 189 */ 190 struct completion done; 191 } unplug; 192 193 /** @reset: Reset related fields. */ 194 struct { 195 /** @wq: Ordered worqueud used to schedule reset operations. */ 196 struct workqueue_struct *wq; 197 198 /** @work: Reset work. */ 199 struct work_struct work; 200 201 /** @pending: Set to true if a reset is pending. */ 202 atomic_t pending; 203 204 /** 205 * @fast: True if the post_reset logic can proceed with a fast reset. 206 * 207 * A fast reset is just a reset where the driver doesn't reload the FW sections. 208 * 209 * Any time the firmware is properly suspended, a fast reset can take place. 210 * On the other hand, if the halt operation failed, the driver will reload 211 * all FW sections to make sure we start from a fresh state. 212 */ 213 bool fast; 214 } reset; 215 216 /** @pm: Power management related data. */ 217 struct { 218 /** @state: Power state. */ 219 atomic_t state; 220 221 /** 222 * @mmio_lock: Lock protecting MMIO userspace CPU mappings. 223 * 224 * This is needed to ensure we map the dummy IO pages when 225 * the device is being suspended, and the real IO pages when 226 * the device is being resumed. We can't just do with the 227 * state atomicity to deal with this race. 228 */ 229 struct mutex mmio_lock; 230 231 /** 232 * @dummy_latest_flush: Dummy LATEST_FLUSH page. 233 * 234 * Used to replace the real LATEST_FLUSH page when the GPU 235 * is suspended. 236 */ 237 struct page *dummy_latest_flush; 238 239 /** @recovery_needed: True when a resume attempt failed. */ 240 atomic_t recovery_needed; 241 } pm; 242 243 /** @profile_mask: User-set profiling flags for job accounting. */ 244 u32 profile_mask; 245 246 /** @fast_rate: Maximum device clock frequency. Set by DVFS */ 247 unsigned long fast_rate; 248 249 #ifdef CONFIG_DEBUG_FS 250 /** @gems: Device-wide list of GEM objects owned by at least one file. */ 251 struct { 252 /** @gems.lock: Protects the device-wide list of GEM objects. */ 253 struct mutex lock; 254 255 /** @node: Used to keep track of all the device's DRM objects */ 256 struct list_head node; 257 } gems; 258 #endif 259 }; 260 261 struct panthor_gpu_usage { 262 u64 time; 263 u64 cycles; 264 }; 265 266 /** 267 * struct panthor_file - Panthor file 268 */ 269 struct panthor_file { 270 /** @ptdev: Device attached to this file. */ 271 struct panthor_device *ptdev; 272 273 /** @user_mmio: User MMIO related fields. */ 274 struct { 275 /** 276 * @offset: Offset used for user MMIO mappings. 277 * 278 * This offset should not be used to check the type of mapping 279 * except in panthor_mmap(). After that point, MMIO mapping 280 * offsets have been adjusted to match 281 * DRM_PANTHOR_USER_MMIO_OFFSET and that macro should be used 282 * instead. 283 * Make sure this rule is followed at all times, because 284 * userspace is in control of the offset, and can change the 285 * value behind our back. Otherwise it can lead to erroneous 286 * branching happening in kernel space. 287 */ 288 u64 offset; 289 } user_mmio; 290 291 /** @vms: VM pool attached to this file. */ 292 struct panthor_vm_pool *vms; 293 294 /** @groups: Scheduling group pool attached to this file. */ 295 struct panthor_group_pool *groups; 296 297 /** @stats: cycle and timestamp measures for job execution. */ 298 struct panthor_gpu_usage stats; 299 }; 300 301 int panthor_device_init(struct panthor_device *ptdev); 302 void panthor_device_unplug(struct panthor_device *ptdev); 303 304 /** 305 * panthor_device_schedule_reset() - Schedules a reset operation 306 */ 307 static inline void panthor_device_schedule_reset(struct panthor_device *ptdev) 308 { 309 if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) && 310 atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE) 311 queue_work(ptdev->reset.wq, &ptdev->reset.work); 312 } 313 314 /** 315 * panthor_device_reset_is_pending() - Checks if a reset is pending. 316 * 317 * Return: true if a reset is pending, false otherwise. 318 */ 319 static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev) 320 { 321 return atomic_read(&ptdev->reset.pending) != 0; 322 } 323 324 int panthor_device_mmap_io(struct panthor_device *ptdev, 325 struct vm_area_struct *vma); 326 327 int panthor_device_resume(struct device *dev); 328 int panthor_device_suspend(struct device *dev); 329 330 static inline int panthor_device_resume_and_get(struct panthor_device *ptdev) 331 { 332 int ret = pm_runtime_resume_and_get(ptdev->base.dev); 333 334 /* If the resume failed, we need to clear the runtime_error, which 335 * can done by forcing the RPM state to suspended. If multiple 336 * threads called panthor_device_resume_and_get(), we only want 337 * one of them to update the state, hence the cmpxchg. Note that a 338 * thread might enter panthor_device_resume_and_get() and call 339 * pm_runtime_resume_and_get() after another thread had attempted 340 * to resume and failed. This means we will end up with an error 341 * without even attempting a resume ourselves. The only risk here 342 * is to report an error when the second resume attempt might have 343 * succeeded. Given resume errors are not expected, this is probably 344 * something we can live with. 345 */ 346 if (ret && atomic_cmpxchg(&ptdev->pm.recovery_needed, 1, 0) == 1) 347 pm_runtime_set_suspended(ptdev->base.dev); 348 349 return ret; 350 } 351 352 enum drm_panthor_exception_type { 353 DRM_PANTHOR_EXCEPTION_OK = 0x00, 354 DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04, 355 DRM_PANTHOR_EXCEPTION_KABOOM = 0x05, 356 DRM_PANTHOR_EXCEPTION_EUREKA = 0x06, 357 DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08, 358 DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f, 359 DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f, 360 DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40, 361 DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41, 362 DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44, 363 DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48, 364 DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49, 365 DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a, 366 DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b, 367 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50, 368 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51, 369 DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55, 370 DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58, 371 DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59, 372 DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a, 373 DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b, 374 DRM_PANTHOR_EXCEPTION_OOM = 0x60, 375 DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68, 376 DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69, 377 DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80, 378 DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88, 379 DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89, 380 DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a, 381 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0, 382 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1, 383 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2, 384 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3, 385 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4, 386 DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8, 387 DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9, 388 DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca, 389 DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb, 390 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9, 391 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda, 392 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb, 393 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0, 394 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4, 395 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5, 396 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6, 397 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7, 398 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8, 399 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9, 400 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea, 401 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb, 402 }; 403 404 /** 405 * panthor_exception_is_fault() - Checks if an exception is a fault. 406 * 407 * Return: true if the exception is a fault, false otherwise. 408 */ 409 static inline bool 410 panthor_exception_is_fault(u32 exception_code) 411 { 412 return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT; 413 } 414 415 const char *panthor_exception_name(struct panthor_device *ptdev, 416 u32 exception_code); 417 418 /** 419 * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt 420 * registration function. 421 * 422 * The boiler-plate to gracefully deal with shared interrupts is 423 * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER() 424 * just after the actual handler. The handler prototype is: 425 * 426 * void (*handler)(struct panthor_device *, u32 status); 427 */ 428 #define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \ 429 static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \ 430 { \ 431 struct panthor_irq *pirq = data; \ 432 struct panthor_device *ptdev = pirq->ptdev; \ 433 enum panthor_irq_state old_state; \ 434 \ 435 if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \ 436 return IRQ_NONE; \ 437 \ 438 guard(spinlock_irqsave)(&pirq->mask_lock); \ 439 old_state = atomic_cmpxchg(&pirq->state, \ 440 PANTHOR_IRQ_STATE_ACTIVE, \ 441 PANTHOR_IRQ_STATE_PROCESSING); \ 442 if (old_state != PANTHOR_IRQ_STATE_ACTIVE) \ 443 return IRQ_NONE; \ 444 \ 445 gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \ 446 return IRQ_WAKE_THREAD; \ 447 } \ 448 \ 449 static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data) \ 450 { \ 451 struct panthor_irq *pirq = data; \ 452 struct panthor_device *ptdev = pirq->ptdev; \ 453 irqreturn_t ret = IRQ_NONE; \ 454 \ 455 while (true) { \ 456 /* It's safe to access pirq->mask without the lock held here. If a new \ 457 * event gets added to the mask and the corresponding IRQ is pending, \ 458 * we'll process it right away instead of adding an extra raw -> threaded \ 459 * round trip. If an event is removed and the status bit is set, it will \ 460 * be ignored, just like it would have been if the mask had been adjusted \ 461 * right before the HW event kicks in. TLDR; it's all expected races we're \ 462 * covered for. \ 463 */ \ 464 u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \ 465 \ 466 if (!status) \ 467 break; \ 468 \ 469 __handler(ptdev, status); \ 470 ret = IRQ_HANDLED; \ 471 } \ 472 \ 473 scoped_guard(spinlock_irqsave, &pirq->mask_lock) { \ 474 enum panthor_irq_state old_state; \ 475 \ 476 old_state = atomic_cmpxchg(&pirq->state, \ 477 PANTHOR_IRQ_STATE_PROCESSING, \ 478 PANTHOR_IRQ_STATE_ACTIVE); \ 479 if (old_state == PANTHOR_IRQ_STATE_PROCESSING) \ 480 gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ 481 } \ 482 \ 483 return ret; \ 484 } \ 485 \ 486 static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \ 487 { \ 488 scoped_guard(spinlock_irqsave, &pirq->mask_lock) { \ 489 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING); \ 490 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ 491 } \ 492 synchronize_irq(pirq->irq); \ 493 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED); \ 494 } \ 495 \ 496 static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq) \ 497 { \ 498 guard(spinlock_irqsave)(&pirq->mask_lock); \ 499 \ 500 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE); \ 501 gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, pirq->mask); \ 502 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ 503 } \ 504 \ 505 static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ 506 struct panthor_irq *pirq, \ 507 int irq, u32 mask) \ 508 { \ 509 pirq->ptdev = ptdev; \ 510 pirq->irq = irq; \ 511 pirq->mask = mask; \ 512 spin_lock_init(&pirq->mask_lock); \ 513 panthor_ ## __name ## _irq_resume(pirq); \ 514 \ 515 return devm_request_threaded_irq(ptdev->base.dev, irq, \ 516 panthor_ ## __name ## _irq_raw_handler, \ 517 panthor_ ## __name ## _irq_threaded_handler, \ 518 IRQF_SHARED, KBUILD_MODNAME "-" # __name, \ 519 pirq); \ 520 } \ 521 \ 522 static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq *pirq, u32 mask) \ 523 { \ 524 guard(spinlock_irqsave)(&pirq->mask_lock); \ 525 pirq->mask |= mask; \ 526 \ 527 /* The only situation where we need to write the new mask is if the IRQ is active. \ 528 * If it's being processed, the mask will be restored for us in _irq_threaded_handler() \ 529 * on the PROCESSING -> ACTIVE transition. \ 530 * If the IRQ is suspended/suspending, the mask is restored at resume time. \ 531 */ \ 532 if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \ 533 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ 534 } \ 535 \ 536 static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\ 537 { \ 538 guard(spinlock_irqsave)(&pirq->mask_lock); \ 539 pirq->mask &= ~mask; \ 540 \ 541 /* The only situation where we need to write the new mask is if the IRQ is active. \ 542 * If it's being processed, the mask will be restored for us in _irq_threaded_handler() \ 543 * on the PROCESSING -> ACTIVE transition. \ 544 * If the IRQ is suspended/suspending, the mask is restored at resume time. \ 545 */ \ 546 if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \ 547 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ 548 } 549 550 extern struct workqueue_struct *panthor_cleanup_wq; 551 552 static inline void gpu_write(struct panthor_device *ptdev, u32 reg, u32 data) 553 { 554 writel(data, ptdev->iomem + reg); 555 } 556 557 static inline u32 gpu_read(struct panthor_device *ptdev, u32 reg) 558 { 559 return readl(ptdev->iomem + reg); 560 } 561 562 static inline u32 gpu_read_relaxed(struct panthor_device *ptdev, u32 reg) 563 { 564 return readl_relaxed(ptdev->iomem + reg); 565 } 566 567 static inline void gpu_write64(struct panthor_device *ptdev, u32 reg, u64 data) 568 { 569 gpu_write(ptdev, reg, lower_32_bits(data)); 570 gpu_write(ptdev, reg + 4, upper_32_bits(data)); 571 } 572 573 static inline u64 gpu_read64(struct panthor_device *ptdev, u32 reg) 574 { 575 return (gpu_read(ptdev, reg) | ((u64)gpu_read(ptdev, reg + 4) << 32)); 576 } 577 578 static inline u64 gpu_read64_relaxed(struct panthor_device *ptdev, u32 reg) 579 { 580 return (gpu_read_relaxed(ptdev, reg) | 581 ((u64)gpu_read_relaxed(ptdev, reg + 4) << 32)); 582 } 583 584 static inline u64 gpu_read64_counter(struct panthor_device *ptdev, u32 reg) 585 { 586 u32 lo, hi1, hi2; 587 do { 588 hi1 = gpu_read(ptdev, reg + 4); 589 lo = gpu_read(ptdev, reg); 590 hi2 = gpu_read(ptdev, reg + 4); 591 } while (hi1 != hi2); 592 return lo | ((u64)hi2 << 32); 593 } 594 595 #define gpu_read_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \ 596 read_poll_timeout(gpu_read, val, cond, delay_us, timeout_us, false, \ 597 dev, reg) 598 599 #define gpu_read_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ 600 timeout_us) \ 601 read_poll_timeout_atomic(gpu_read, val, cond, delay_us, timeout_us, \ 602 false, dev, reg) 603 604 #define gpu_read64_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \ 605 read_poll_timeout(gpu_read64, val, cond, delay_us, timeout_us, false, \ 606 dev, reg) 607 608 #define gpu_read64_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ 609 timeout_us) \ 610 read_poll_timeout_atomic(gpu_read64, val, cond, delay_us, timeout_us, \ 611 false, dev, reg) 612 613 #define gpu_read_relaxed_poll_timeout_atomic(dev, reg, val, cond, delay_us, \ 614 timeout_us) \ 615 read_poll_timeout_atomic(gpu_read_relaxed, val, cond, delay_us, \ 616 timeout_us, false, dev, reg) 617 618 #define gpu_read64_relaxed_poll_timeout(dev, reg, val, cond, delay_us, \ 619 timeout_us) \ 620 read_poll_timeout(gpu_read64_relaxed, val, cond, delay_us, timeout_us, \ 621 false, dev, reg) 622 623 #endif 624