1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2025 ARM Limited. All rights reserved. */ 3 4 #include <linux/platform_device.h> 5 #include <linux/interrupt.h> 6 #include <linux/cleanup.h> 7 #include <linux/iopoll.h> 8 #include <linux/wait.h> 9 10 #include <drm/drm_managed.h> 11 #include <drm/drm_print.h> 12 13 #include "panthor_device.h" 14 #include "panthor_gpu_regs.h" 15 #include "panthor_hw.h" 16 #include "panthor_pwr.h" 17 #include "panthor_pwr_regs.h" 18 19 #define PWR_INTERRUPTS_MASK \ 20 (PWR_IRQ_POWER_CHANGED_SINGLE | \ 21 PWR_IRQ_POWER_CHANGED_ALL | \ 22 PWR_IRQ_DELEGATION_CHANGED | \ 23 PWR_IRQ_RESET_COMPLETED | \ 24 PWR_IRQ_RETRACT_COMPLETED | \ 25 PWR_IRQ_INSPECT_COMPLETED | \ 26 PWR_IRQ_COMMAND_NOT_ALLOWED | \ 27 PWR_IRQ_COMMAND_INVALID) 28 29 #define PWR_ALL_CORES_MASK GENMASK_U64(63, 0) 30 31 #define PWR_DOMAIN_MAX_BITS 16 32 33 #define PWR_TRANSITION_TIMEOUT_US (2ULL * USEC_PER_SEC) 34 35 #define PWR_RETRACT_TIMEOUT_US (2ULL * USEC_PER_MSEC) 36 37 #define PWR_RESET_TIMEOUT_MS 500 38 39 /** 40 * struct panthor_pwr - PWR_CONTROL block management data. 41 */ 42 struct panthor_pwr { 43 /** @iomem: CPU mapping of PWR_CONTROL iomem region */ 44 void __iomem *iomem; 45 46 /** @irq: PWR irq. */ 47 struct panthor_irq irq; 48 49 /** @reqs_lock: Lock protecting access to pending_reqs. */ 50 spinlock_t reqs_lock; 51 52 /** @pending_reqs: Pending PWR requests. */ 53 u32 pending_reqs; 54 55 /** @reqs_acked: PWR request wait queue. */ 56 wait_queue_head_t reqs_acked; 57 }; 58 59 static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status) 60 { 61 struct panthor_pwr *pwr = ptdev->pwr; 62 63 spin_lock(&ptdev->pwr->reqs_lock); 64 gpu_write(pwr->irq.iomem, INT_CLEAR, status); 65 66 if (unlikely(status & PWR_IRQ_COMMAND_NOT_ALLOWED)) 67 drm_err(&ptdev->base, "PWR_IRQ: COMMAND_NOT_ALLOWED"); 68 69 if (unlikely(status & PWR_IRQ_COMMAND_INVALID)) 70 drm_err(&ptdev->base, "PWR_IRQ: COMMAND_INVALID"); 71 72 if (status & ptdev->pwr->pending_reqs) { 73 ptdev->pwr->pending_reqs &= ~status; 74 wake_up_all(&ptdev->pwr->reqs_acked); 75 } 76 spin_unlock(&ptdev->pwr->reqs_lock); 77 } 78 PANTHOR_IRQ_HANDLER(pwr, panthor_pwr_irq_handler); 79 80 static void panthor_pwr_write_command(struct panthor_device *ptdev, u32 command, u64 args) 81 { 82 struct panthor_pwr *pwr = ptdev->pwr; 83 84 if (args) 85 gpu_write64(pwr->iomem, PWR_CMDARG, args); 86 87 gpu_write(pwr->iomem, PWR_COMMAND, command); 88 } 89 90 static bool reset_irq_raised(struct panthor_device *ptdev) 91 { 92 struct panthor_pwr *pwr = ptdev->pwr; 93 94 return gpu_read(pwr->irq.iomem, INT_RAWSTAT) & PWR_IRQ_RESET_COMPLETED; 95 } 96 97 static bool reset_pending(struct panthor_device *ptdev) 98 { 99 return (ptdev->pwr->pending_reqs & PWR_IRQ_RESET_COMPLETED); 100 } 101 102 static int panthor_pwr_reset(struct panthor_device *ptdev, u32 reset_cmd) 103 { 104 struct panthor_pwr *pwr = ptdev->pwr; 105 106 scoped_guard(spinlock_irqsave, &ptdev->pwr->reqs_lock) { 107 if (reset_pending(ptdev)) { 108 drm_WARN(&ptdev->base, 1, "Reset already pending"); 109 } else { 110 ptdev->pwr->pending_reqs |= PWR_IRQ_RESET_COMPLETED; 111 gpu_write(pwr->irq.iomem, INT_CLEAR, PWR_IRQ_RESET_COMPLETED); 112 panthor_pwr_write_command(ptdev, reset_cmd, 0); 113 } 114 } 115 116 if (!wait_event_timeout(ptdev->pwr->reqs_acked, !reset_pending(ptdev), 117 msecs_to_jiffies(PWR_RESET_TIMEOUT_MS))) { 118 guard(spinlock_irqsave)(&ptdev->pwr->reqs_lock); 119 120 if (reset_pending(ptdev) && !reset_irq_raised(ptdev)) { 121 drm_err(&ptdev->base, "RESET timed out (0x%x)", reset_cmd); 122 return -ETIMEDOUT; 123 } 124 125 ptdev->pwr->pending_reqs &= ~PWR_IRQ_RESET_COMPLETED; 126 } 127 128 return 0; 129 } 130 131 static const char *get_domain_name(u8 domain) 132 { 133 switch (domain) { 134 case PWR_COMMAND_DOMAIN_L2: 135 return "L2"; 136 case PWR_COMMAND_DOMAIN_TILER: 137 return "Tiler"; 138 case PWR_COMMAND_DOMAIN_SHADER: 139 return "Shader"; 140 case PWR_COMMAND_DOMAIN_BASE: 141 return "Base"; 142 case PWR_COMMAND_DOMAIN_STACK: 143 return "Stack"; 144 } 145 return "Unknown"; 146 } 147 148 static u32 get_domain_base(u8 domain) 149 { 150 switch (domain) { 151 case PWR_COMMAND_DOMAIN_L2: 152 return PWR_L2_PRESENT; 153 case PWR_COMMAND_DOMAIN_TILER: 154 return PWR_TILER_PRESENT; 155 case PWR_COMMAND_DOMAIN_SHADER: 156 return PWR_SHADER_PRESENT; 157 case PWR_COMMAND_DOMAIN_BASE: 158 return PWR_BASE_PRESENT; 159 case PWR_COMMAND_DOMAIN_STACK: 160 return PWR_STACK_PRESENT; 161 } 162 return 0; 163 } 164 165 static u32 get_domain_ready_reg(u32 domain) 166 { 167 return get_domain_base(domain) + (PWR_L2_READY - PWR_L2_PRESENT); 168 } 169 170 static u32 get_domain_pwrtrans_reg(u32 domain) 171 { 172 return get_domain_base(domain) + (PWR_L2_PWRTRANS - PWR_L2_PRESENT); 173 } 174 175 static bool is_valid_domain(u32 domain) 176 { 177 return get_domain_base(domain) != 0; 178 } 179 180 static bool has_rtu(struct panthor_device *ptdev) 181 { 182 return ptdev->gpu_info.gpu_features & GPU_FEATURES_RAY_TRAVERSAL; 183 } 184 185 static u8 get_domain_subdomain(struct panthor_device *ptdev, u32 domain) 186 { 187 if (domain == PWR_COMMAND_DOMAIN_SHADER && has_rtu(ptdev)) 188 return PWR_COMMAND_SUBDOMAIN_RTU; 189 190 return 0; 191 } 192 193 static int panthor_pwr_domain_wait_transition(struct panthor_device *ptdev, u32 domain, 194 u32 timeout_us) 195 { 196 struct panthor_pwr *pwr = ptdev->pwr; 197 u32 pwrtrans_reg = get_domain_pwrtrans_reg(domain); 198 u64 val; 199 int ret = 0; 200 201 ret = gpu_read64_poll_timeout(pwr->iomem, pwrtrans_reg, val, !(PWR_ALL_CORES_MASK & val), 100, 202 timeout_us); 203 if (ret) { 204 drm_err(&ptdev->base, "%s domain power in transition, pwrtrans(0x%llx)", 205 get_domain_name(domain), val); 206 return ret; 207 } 208 209 return 0; 210 } 211 212 static void panthor_pwr_debug_info_show(struct panthor_device *ptdev) 213 { 214 struct panthor_pwr *pwr = ptdev->pwr; 215 216 drm_info(&ptdev->base, "GPU_FEATURES: 0x%016llx", ptdev->gpu_info.gpu_features); 217 drm_info(&ptdev->base, "PWR_STATUS: 0x%016llx", gpu_read64(pwr->iomem, PWR_STATUS)); 218 drm_info(&ptdev->base, "L2_PRESENT: 0x%016llx", gpu_read64(pwr->iomem, PWR_L2_PRESENT)); 219 drm_info(&ptdev->base, "L2_PWRTRANS: 0x%016llx", gpu_read64(pwr->iomem, PWR_L2_PWRTRANS)); 220 drm_info(&ptdev->base, "L2_READY: 0x%016llx", gpu_read64(pwr->iomem, PWR_L2_READY)); 221 drm_info(&ptdev->base, "TILER_PRESENT: 0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_PRESENT)); 222 drm_info(&ptdev->base, "TILER_PWRTRANS: 0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_PWRTRANS)); 223 drm_info(&ptdev->base, "TILER_READY: 0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_READY)); 224 drm_info(&ptdev->base, "SHADER_PRESENT: 0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_PRESENT)); 225 drm_info(&ptdev->base, "SHADER_PWRTRANS: 0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_PWRTRANS)); 226 drm_info(&ptdev->base, "SHADER_READY: 0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_READY)); 227 } 228 229 static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd, u32 domain, 230 u64 mask, u32 timeout_us) 231 { 232 struct panthor_pwr *pwr = ptdev->pwr; 233 u32 ready_reg = get_domain_ready_reg(domain); 234 u32 pwr_cmd = PWR_COMMAND_DEF(cmd, domain, get_domain_subdomain(ptdev, domain)); 235 u64 expected_val = 0; 236 u64 val; 237 int ret = 0; 238 239 if (drm_WARN_ON(&ptdev->base, !is_valid_domain(domain))) 240 return -EINVAL; 241 242 switch (cmd) { 243 case PWR_COMMAND_POWER_DOWN: 244 expected_val = 0; 245 break; 246 case PWR_COMMAND_POWER_UP: 247 expected_val = mask; 248 break; 249 default: 250 drm_err(&ptdev->base, "Invalid power domain transition command (0x%x)", cmd); 251 return -EINVAL; 252 } 253 254 ret = panthor_pwr_domain_wait_transition(ptdev, domain, timeout_us); 255 if (ret) 256 return ret; 257 258 /* domain already in target state, return early */ 259 if ((gpu_read64(pwr->iomem, ready_reg) & mask) == expected_val) 260 return 0; 261 262 panthor_pwr_write_command(ptdev, pwr_cmd, mask); 263 264 ret = gpu_read64_poll_timeout(pwr->iomem, ready_reg, val, (mask & val) == expected_val, 265 100, timeout_us); 266 if (ret) { 267 drm_err(&ptdev->base, 268 "timeout waiting on %s power domain transition, cmd(0x%x), arg(0x%llx)", 269 get_domain_name(domain), pwr_cmd, mask); 270 panthor_pwr_debug_info_show(ptdev); 271 return ret; 272 } 273 274 return 0; 275 } 276 277 #define panthor_pwr_domain_power_off(__ptdev, __domain, __mask, __timeout_us) \ 278 panthor_pwr_domain_transition(__ptdev, PWR_COMMAND_POWER_DOWN, __domain, __mask, \ 279 __timeout_us) 280 281 #define panthor_pwr_domain_power_on(__ptdev, __domain, __mask, __timeout_us) \ 282 panthor_pwr_domain_transition(__ptdev, PWR_COMMAND_POWER_UP, __domain, __mask, __timeout_us) 283 284 /** 285 * retract_domain() - Retract control of a domain from MCU 286 * @ptdev: Device. 287 * @domain: Domain to retract the control 288 * 289 * Retracting L2 domain is not expected since it won't be delegated. 290 * 291 * Return: 0 on success or retracted already. 292 * -EPERM if domain is L2. 293 * A negative error code otherwise. 294 */ 295 static int retract_domain(struct panthor_device *ptdev, u32 domain) 296 { 297 struct panthor_pwr *pwr = ptdev->pwr; 298 const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_RETRACT, domain, 0); 299 const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS); 300 const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain); 301 const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain); 302 u64 val; 303 int ret; 304 305 if (drm_WARN_ON(&ptdev->base, domain == PWR_COMMAND_DOMAIN_L2)) 306 return -EPERM; 307 308 ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val, 309 !(PWR_STATUS_RETRACT_PENDING & val), 0, 310 PWR_RETRACT_TIMEOUT_US); 311 if (ret) { 312 drm_err(&ptdev->base, "%s domain retract pending", get_domain_name(domain)); 313 return ret; 314 } 315 316 if (!(pwr_status & delegated_mask)) { 317 drm_dbg(&ptdev->base, "%s domain already retracted", get_domain_name(domain)); 318 return 0; 319 } 320 321 panthor_pwr_write_command(ptdev, pwr_cmd, 0); 322 323 /* 324 * On successful retraction 325 * allow-flag will be set with delegated-flag being cleared. 326 */ 327 ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val, 328 ((delegated_mask | allow_mask) & val) == allow_mask, 10, 329 PWR_TRANSITION_TIMEOUT_US); 330 if (ret) { 331 drm_err(&ptdev->base, "Retracting %s domain timeout, cmd(0x%x)", 332 get_domain_name(domain), pwr_cmd); 333 return ret; 334 } 335 336 return 0; 337 } 338 339 /** 340 * delegate_domain() - Delegate control of a domain to MCU 341 * @ptdev: Device. 342 * @domain: Domain to delegate the control 343 * 344 * Delegating L2 domain is prohibited. 345 * 346 * Return: 347 * * 0 on success or delegated already. 348 * * -EPERM if domain is L2. 349 * * A negative error code otherwise. 350 */ 351 static int delegate_domain(struct panthor_device *ptdev, u32 domain) 352 { 353 struct panthor_pwr *pwr = ptdev->pwr; 354 const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_DELEGATE, domain, 0); 355 const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS); 356 const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain); 357 const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain); 358 u64 val; 359 int ret; 360 361 if (drm_WARN_ON(&ptdev->base, domain == PWR_COMMAND_DOMAIN_L2)) 362 return -EPERM; 363 364 /* Already delegated, exit early */ 365 if (pwr_status & delegated_mask) 366 return 0; 367 368 /* Check if the command is allowed before delegating. */ 369 if (!(pwr_status & allow_mask)) { 370 drm_warn(&ptdev->base, "Delegating %s domain not allowed", get_domain_name(domain)); 371 return -EPERM; 372 } 373 374 ret = panthor_pwr_domain_wait_transition(ptdev, domain, PWR_TRANSITION_TIMEOUT_US); 375 if (ret) 376 return ret; 377 378 panthor_pwr_write_command(ptdev, pwr_cmd, 0); 379 380 /* 381 * On successful delegation 382 * allow-flag will be cleared with delegated-flag being set. 383 */ 384 ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val, 385 ((delegated_mask | allow_mask) & val) == delegated_mask, 386 10, PWR_TRANSITION_TIMEOUT_US); 387 if (ret) { 388 drm_err(&ptdev->base, "Delegating %s domain timeout, cmd(0x%x)", 389 get_domain_name(domain), pwr_cmd); 390 return ret; 391 } 392 393 return 0; 394 } 395 396 static int panthor_pwr_delegate_domains(struct panthor_device *ptdev) 397 { 398 int ret; 399 400 if (!ptdev->pwr) 401 return 0; 402 403 ret = delegate_domain(ptdev, PWR_COMMAND_DOMAIN_SHADER); 404 if (ret) 405 return ret; 406 407 ret = delegate_domain(ptdev, PWR_COMMAND_DOMAIN_TILER); 408 if (ret) 409 goto err_retract_shader; 410 411 return 0; 412 413 err_retract_shader: 414 retract_domain(ptdev, PWR_COMMAND_DOMAIN_SHADER); 415 416 return ret; 417 } 418 419 /** 420 * panthor_pwr_domain_force_off - Forcefully power down a domain. 421 * @ptdev: Device. 422 * @domain: Domain to forcefully power down. 423 * 424 * This function will attempt to retract and power off the requested power 425 * domain. However, if retraction fails, the operation is aborted. If power off 426 * fails, the domain will remain retracted and under the host control. 427 * 428 * Return: 0 on success or a negative error code on failure. 429 */ 430 static int panthor_pwr_domain_force_off(struct panthor_device *ptdev, u32 domain) 431 { 432 struct panthor_pwr *pwr = ptdev->pwr; 433 const u64 domain_ready = gpu_read64(pwr->iomem, get_domain_ready_reg(domain)); 434 int ret; 435 436 /* Domain already powered down, early exit. */ 437 if (!domain_ready) 438 return 0; 439 440 /* Domain has to be in host control to issue power off command. */ 441 ret = retract_domain(ptdev, domain); 442 if (ret) 443 return ret; 444 445 return panthor_pwr_domain_power_off(ptdev, domain, domain_ready, PWR_TRANSITION_TIMEOUT_US); 446 } 447 448 void panthor_pwr_unplug(struct panthor_device *ptdev) 449 { 450 unsigned long flags; 451 452 if (!ptdev->pwr) 453 return; 454 455 /* Make sure the IRQ handler is not running after that point. */ 456 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) 457 panthor_pwr_irq_suspend(&ptdev->pwr->irq); 458 459 /* Wake-up all waiters. */ 460 spin_lock_irqsave(&ptdev->pwr->reqs_lock, flags); 461 ptdev->pwr->pending_reqs = 0; 462 wake_up_all(&ptdev->pwr->reqs_acked); 463 spin_unlock_irqrestore(&ptdev->pwr->reqs_lock, flags); 464 } 465 466 int panthor_pwr_init(struct panthor_device *ptdev) 467 { 468 struct panthor_pwr *pwr; 469 int err, irq; 470 471 if (!panthor_hw_has_pwr_ctrl(ptdev)) 472 return 0; 473 474 pwr = drmm_kzalloc(&ptdev->base, sizeof(*pwr), GFP_KERNEL); 475 if (!pwr) 476 return -ENOMEM; 477 478 pwr->iomem = ptdev->iomem + PWR_CONTROL_BASE; 479 spin_lock_init(&pwr->reqs_lock); 480 init_waitqueue_head(&pwr->reqs_acked); 481 ptdev->pwr = pwr; 482 483 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); 484 if (irq < 0) 485 return irq; 486 487 err = panthor_request_pwr_irq(ptdev, &pwr->irq, irq, 488 pwr->iomem + PWR_INT_BASE); 489 if (err) 490 return err; 491 492 panthor_pwr_irq_enable_events(&pwr->irq, PWR_INTERRUPTS_MASK); 493 panthor_pwr_irq_resume(&pwr->irq); 494 return 0; 495 } 496 497 int panthor_pwr_reset_soft(struct panthor_device *ptdev) 498 { 499 struct panthor_pwr *pwr = ptdev->pwr; 500 501 if (!(gpu_read64(pwr->iomem, PWR_STATUS) & PWR_STATUS_ALLOW_SOFT_RESET)) { 502 drm_err(&ptdev->base, "RESET_SOFT not allowed"); 503 return -EOPNOTSUPP; 504 } 505 506 return panthor_pwr_reset(ptdev, PWR_COMMAND_RESET_SOFT); 507 } 508 509 void panthor_pwr_l2_power_off(struct panthor_device *ptdev) 510 { 511 struct panthor_pwr *pwr = ptdev->pwr; 512 const u64 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2); 513 const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS); 514 515 /* Abort if L2 power off constraints are not satisfied */ 516 if (!(pwr_status & l2_allow_mask)) { 517 drm_warn(&ptdev->base, "Power off L2 domain not allowed"); 518 return; 519 } 520 521 /* It is expected that when halting the MCU, it would power down its 522 * delegated domains. However, an unresponsive or hung MCU may not do 523 * so, which is why we need to check and retract the domains back into 524 * host control to be powered down in the right order before powering 525 * down the L2. 526 */ 527 if (panthor_pwr_domain_force_off(ptdev, PWR_COMMAND_DOMAIN_TILER)) 528 return; 529 530 if (panthor_pwr_domain_force_off(ptdev, PWR_COMMAND_DOMAIN_SHADER)) 531 return; 532 533 panthor_pwr_domain_power_off(ptdev, PWR_COMMAND_DOMAIN_L2, ptdev->gpu_info.l2_present, 534 PWR_TRANSITION_TIMEOUT_US); 535 } 536 537 int panthor_pwr_l2_power_on(struct panthor_device *ptdev) 538 { 539 struct panthor_pwr *pwr = ptdev->pwr; 540 const u32 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS); 541 const u32 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2); 542 int ret; 543 544 if ((pwr_status & l2_allow_mask) == 0) { 545 drm_warn(&ptdev->base, "Power on L2 domain not allowed"); 546 return -EPERM; 547 } 548 549 ret = panthor_pwr_domain_power_on(ptdev, PWR_COMMAND_DOMAIN_L2, ptdev->gpu_info.l2_present, 550 PWR_TRANSITION_TIMEOUT_US); 551 if (ret) 552 return ret; 553 554 /* Delegate control of the shader and tiler power domains to the MCU as 555 * it can better manage which shader/tiler cores need to be powered up 556 * or can be powered down based on currently running jobs. 557 * 558 * If the shader and tiler domains are already delegated to the MCU, 559 * this call would just return early. 560 */ 561 return panthor_pwr_delegate_domains(ptdev); 562 } 563 564 void panthor_pwr_suspend(struct panthor_device *ptdev) 565 { 566 if (!ptdev->pwr) 567 return; 568 569 panthor_pwr_irq_suspend(&ptdev->pwr->irq); 570 } 571 572 void panthor_pwr_resume(struct panthor_device *ptdev) 573 { 574 if (!ptdev->pwr) 575 return; 576 577 panthor_pwr_irq_resume(&ptdev->pwr->irq); 578 } 579