1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 4 5 #include <linux/clk.h> 6 #include <linux/reset.h> 7 #include <linux/platform_device.h> 8 #include <linux/pm_domain.h> 9 #include <linux/pm_runtime.h> 10 #include <linux/regulator/consumer.h> 11 12 #include "panfrost_device.h" 13 #include "panfrost_devfreq.h" 14 #include "panfrost_features.h" 15 #include "panfrost_gem.h" 16 #include "panfrost_issues.h" 17 #include "panfrost_gpu.h" 18 #include "panfrost_job.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_perfcnt.h" 21 22 static int panfrost_reset_init(struct panfrost_device *pfdev) 23 { 24 pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->base.dev); 25 if (IS_ERR(pfdev->rstc)) { 26 dev_err(pfdev->base.dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc)); 27 return PTR_ERR(pfdev->rstc); 28 } 29 30 return reset_control_deassert(pfdev->rstc); 31 } 32 33 static void panfrost_reset_fini(struct panfrost_device *pfdev) 34 { 35 reset_control_assert(pfdev->rstc); 36 } 37 38 static int panfrost_clk_init(struct panfrost_device *pfdev) 39 { 40 int err; 41 unsigned long rate; 42 43 pfdev->clock = devm_clk_get(pfdev->base.dev, NULL); 44 if (IS_ERR(pfdev->clock)) { 45 dev_err(pfdev->base.dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock)); 46 return PTR_ERR(pfdev->clock); 47 } 48 49 rate = clk_get_rate(pfdev->clock); 50 dev_info(pfdev->base.dev, "clock rate = %lu\n", rate); 51 52 err = clk_prepare_enable(pfdev->clock); 53 if (err) 54 return err; 55 56 pfdev->bus_clock = devm_clk_get_optional(pfdev->base.dev, "bus"); 57 if (IS_ERR(pfdev->bus_clock)) { 58 dev_err(pfdev->base.dev, "get bus_clock failed %ld\n", 59 PTR_ERR(pfdev->bus_clock)); 60 err = PTR_ERR(pfdev->bus_clock); 61 goto disable_clock; 62 } 63 64 if (pfdev->bus_clock) { 65 rate = clk_get_rate(pfdev->bus_clock); 66 dev_info(pfdev->base.dev, "bus_clock rate = %lu\n", rate); 67 68 err = clk_prepare_enable(pfdev->bus_clock); 69 if (err) 70 goto disable_clock; 71 } 72 73 return 0; 74 75 disable_clock: 76 clk_disable_unprepare(pfdev->clock); 77 78 return err; 79 } 80 81 static void panfrost_clk_fini(struct panfrost_device *pfdev) 82 { 83 clk_disable_unprepare(pfdev->bus_clock); 84 clk_disable_unprepare(pfdev->clock); 85 } 86 87 static int panfrost_regulator_init(struct panfrost_device *pfdev) 88 { 89 int ret, i; 90 91 pfdev->regulators = devm_kcalloc(pfdev->base.dev, pfdev->comp->num_supplies, 92 sizeof(*pfdev->regulators), 93 GFP_KERNEL); 94 if (!pfdev->regulators) 95 return -ENOMEM; 96 97 for (i = 0; i < pfdev->comp->num_supplies; i++) 98 pfdev->regulators[i].supply = pfdev->comp->supply_names[i]; 99 100 ret = devm_regulator_bulk_get(pfdev->base.dev, 101 pfdev->comp->num_supplies, 102 pfdev->regulators); 103 if (ret < 0) { 104 if (ret != -EPROBE_DEFER) 105 dev_err(pfdev->base.dev, "failed to get regulators: %d\n", 106 ret); 107 return ret; 108 } 109 110 ret = regulator_bulk_enable(pfdev->comp->num_supplies, 111 pfdev->regulators); 112 if (ret < 0) { 113 dev_err(pfdev->base.dev, "failed to enable regulators: %d\n", ret); 114 return ret; 115 } 116 117 return 0; 118 } 119 120 static void panfrost_regulator_fini(struct panfrost_device *pfdev) 121 { 122 if (!pfdev->regulators) 123 return; 124 125 regulator_bulk_disable(pfdev->comp->num_supplies, pfdev->regulators); 126 } 127 128 static void panfrost_pm_domain_fini(struct panfrost_device *pfdev) 129 { 130 int i; 131 132 for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) { 133 if (!pfdev->pm_domain_devs[i]) 134 break; 135 136 if (pfdev->pm_domain_links[i]) 137 device_link_del(pfdev->pm_domain_links[i]); 138 139 dev_pm_domain_detach(pfdev->pm_domain_devs[i], true); 140 } 141 } 142 143 static int panfrost_pm_domain_init(struct panfrost_device *pfdev) 144 { 145 int err; 146 int i, num_domains; 147 148 num_domains = of_count_phandle_with_args(pfdev->base.dev->of_node, 149 "power-domains", 150 "#power-domain-cells"); 151 152 /* 153 * Single domain is handled by the core, and, if only a single power 154 * the power domain is requested, the property is optional. 155 */ 156 if (num_domains < 2 && pfdev->comp->num_pm_domains < 2) 157 return 0; 158 159 if (num_domains != pfdev->comp->num_pm_domains) { 160 dev_err(pfdev->base.dev, 161 "Incorrect number of power domains: %d provided, %d needed\n", 162 num_domains, pfdev->comp->num_pm_domains); 163 return -EINVAL; 164 } 165 166 if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs), 167 "Too many supplies in compatible structure.\n")) 168 return -EINVAL; 169 170 for (i = 0; i < num_domains; i++) { 171 pfdev->pm_domain_devs[i] = 172 dev_pm_domain_attach_by_name(pfdev->base.dev, 173 pfdev->comp->pm_domain_names[i]); 174 if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) { 175 err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA; 176 pfdev->pm_domain_devs[i] = NULL; 177 dev_err(pfdev->base.dev, 178 "failed to get pm-domain %s(%d): %d\n", 179 pfdev->comp->pm_domain_names[i], i, err); 180 goto err; 181 } 182 183 pfdev->pm_domain_links[i] = 184 device_link_add(pfdev->base.dev, 185 pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME | 186 DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE); 187 if (!pfdev->pm_domain_links[i]) { 188 dev_err(pfdev->pm_domain_devs[i], 189 "adding device link failed!\n"); 190 err = -ENODEV; 191 goto err; 192 } 193 } 194 195 return 0; 196 197 err: 198 panfrost_pm_domain_fini(pfdev); 199 return err; 200 } 201 202 int panfrost_device_init(struct panfrost_device *pfdev) 203 { 204 int err; 205 206 mutex_init(&pfdev->sched_lock); 207 INIT_LIST_HEAD(&pfdev->scheduled_jobs); 208 INIT_LIST_HEAD(&pfdev->as_lru_list); 209 210 spin_lock_init(&pfdev->as_lock); 211 212 spin_lock_init(&pfdev->cycle_counter.lock); 213 214 #ifdef CONFIG_DEBUG_FS 215 mutex_init(&pfdev->debugfs.gems_lock); 216 INIT_LIST_HEAD(&pfdev->debugfs.gems_list); 217 #endif 218 219 err = panfrost_pm_domain_init(pfdev); 220 if (err) 221 return err; 222 223 err = panfrost_reset_init(pfdev); 224 if (err) { 225 dev_err(pfdev->base.dev, "reset init failed %d\n", err); 226 goto out_pm_domain; 227 } 228 229 err = panfrost_clk_init(pfdev); 230 if (err) { 231 dev_err(pfdev->base.dev, "clk init failed %d\n", err); 232 goto out_reset; 233 } 234 235 err = panfrost_devfreq_init(pfdev); 236 if (err) { 237 if (err != -EPROBE_DEFER) 238 dev_err(pfdev->base.dev, "devfreq init failed %d\n", err); 239 goto out_clk; 240 } 241 242 /* OPP will handle regulators */ 243 if (!pfdev->pfdevfreq.opp_of_table_added) { 244 err = panfrost_regulator_init(pfdev); 245 if (err) 246 goto out_devfreq; 247 } 248 249 pfdev->iomem = devm_platform_ioremap_resource(to_platform_device(pfdev->base.dev), 0); 250 if (IS_ERR(pfdev->iomem)) { 251 err = PTR_ERR(pfdev->iomem); 252 goto out_regulator; 253 } 254 255 err = panfrost_gpu_init(pfdev); 256 if (err) 257 goto out_regulator; 258 259 err = panfrost_mmu_init(pfdev); 260 if (err) 261 goto out_gpu; 262 263 err = panfrost_jm_init(pfdev); 264 if (err) 265 goto out_mmu; 266 267 err = panfrost_perfcnt_init(pfdev); 268 if (err) 269 goto out_job; 270 271 panfrost_gem_init(pfdev); 272 273 return 0; 274 out_job: 275 panfrost_jm_fini(pfdev); 276 out_mmu: 277 panfrost_mmu_fini(pfdev); 278 out_gpu: 279 panfrost_gpu_fini(pfdev); 280 out_regulator: 281 panfrost_regulator_fini(pfdev); 282 out_devfreq: 283 panfrost_devfreq_fini(pfdev); 284 out_clk: 285 panfrost_clk_fini(pfdev); 286 out_reset: 287 panfrost_reset_fini(pfdev); 288 out_pm_domain: 289 panfrost_pm_domain_fini(pfdev); 290 return err; 291 } 292 293 void panfrost_device_fini(struct panfrost_device *pfdev) 294 { 295 panfrost_perfcnt_fini(pfdev); 296 panfrost_jm_fini(pfdev); 297 panfrost_mmu_fini(pfdev); 298 panfrost_gpu_fini(pfdev); 299 panfrost_devfreq_fini(pfdev); 300 panfrost_regulator_fini(pfdev); 301 panfrost_clk_fini(pfdev); 302 panfrost_reset_fini(pfdev); 303 panfrost_pm_domain_fini(pfdev); 304 } 305 306 #define PANFROST_EXCEPTION(id) \ 307 [DRM_PANFROST_EXCEPTION_ ## id] = { \ 308 .name = #id, \ 309 } 310 311 struct panfrost_exception_info { 312 const char *name; 313 }; 314 315 static const struct panfrost_exception_info panfrost_exception_infos[] = { 316 PANFROST_EXCEPTION(OK), 317 PANFROST_EXCEPTION(DONE), 318 PANFROST_EXCEPTION(INTERRUPTED), 319 PANFROST_EXCEPTION(STOPPED), 320 PANFROST_EXCEPTION(TERMINATED), 321 PANFROST_EXCEPTION(KABOOM), 322 PANFROST_EXCEPTION(EUREKA), 323 PANFROST_EXCEPTION(ACTIVE), 324 PANFROST_EXCEPTION(JOB_CONFIG_FAULT), 325 PANFROST_EXCEPTION(JOB_POWER_FAULT), 326 PANFROST_EXCEPTION(JOB_READ_FAULT), 327 PANFROST_EXCEPTION(JOB_WRITE_FAULT), 328 PANFROST_EXCEPTION(JOB_AFFINITY_FAULT), 329 PANFROST_EXCEPTION(JOB_BUS_FAULT), 330 PANFROST_EXCEPTION(INSTR_INVALID_PC), 331 PANFROST_EXCEPTION(INSTR_INVALID_ENC), 332 PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH), 333 PANFROST_EXCEPTION(INSTR_OPERAND_FAULT), 334 PANFROST_EXCEPTION(INSTR_TLS_FAULT), 335 PANFROST_EXCEPTION(INSTR_BARRIER_FAULT), 336 PANFROST_EXCEPTION(INSTR_ALIGN_FAULT), 337 PANFROST_EXCEPTION(DATA_INVALID_FAULT), 338 PANFROST_EXCEPTION(TILE_RANGE_FAULT), 339 PANFROST_EXCEPTION(ADDR_RANGE_FAULT), 340 PANFROST_EXCEPTION(IMPRECISE_FAULT), 341 PANFROST_EXCEPTION(OOM), 342 PANFROST_EXCEPTION(OOM_AFBC), 343 PANFROST_EXCEPTION(UNKNOWN), 344 PANFROST_EXCEPTION(DELAYED_BUS_FAULT), 345 PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT), 346 PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT), 347 PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT), 348 PANFROST_EXCEPTION(TRANSLATION_FAULT_0), 349 PANFROST_EXCEPTION(TRANSLATION_FAULT_1), 350 PANFROST_EXCEPTION(TRANSLATION_FAULT_2), 351 PANFROST_EXCEPTION(TRANSLATION_FAULT_3), 352 PANFROST_EXCEPTION(TRANSLATION_FAULT_4), 353 PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY), 354 PANFROST_EXCEPTION(PERM_FAULT_0), 355 PANFROST_EXCEPTION(PERM_FAULT_1), 356 PANFROST_EXCEPTION(PERM_FAULT_2), 357 PANFROST_EXCEPTION(PERM_FAULT_3), 358 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0), 359 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1), 360 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2), 361 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3), 362 PANFROST_EXCEPTION(ACCESS_FLAG_0), 363 PANFROST_EXCEPTION(ACCESS_FLAG_1), 364 PANFROST_EXCEPTION(ACCESS_FLAG_2), 365 PANFROST_EXCEPTION(ACCESS_FLAG_3), 366 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0), 367 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1), 368 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2), 369 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3), 370 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0), 371 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1), 372 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2), 373 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3), 374 PANFROST_EXCEPTION(MEM_ATTR_FAULT_0), 375 PANFROST_EXCEPTION(MEM_ATTR_FAULT_1), 376 PANFROST_EXCEPTION(MEM_ATTR_FAULT_2), 377 PANFROST_EXCEPTION(MEM_ATTR_FAULT_3), 378 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0), 379 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1), 380 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2), 381 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3), 382 }; 383 384 const char *panfrost_exception_name(u32 exception_code) 385 { 386 if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) || 387 !panfrost_exception_infos[exception_code].name)) 388 return "Unknown exception type"; 389 390 return panfrost_exception_infos[exception_code].name; 391 } 392 393 bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev, 394 u32 exception_code) 395 { 396 /* If an occlusion query write causes a bus fault on affected GPUs, 397 * future fragment jobs may hang. Reset to workaround. 398 */ 399 if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT) 400 return panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_3076); 401 402 /* No other GPUs we support need a reset */ 403 return false; 404 } 405 406 void panfrost_device_reset(struct panfrost_device *pfdev, bool enable_job_int) 407 { 408 panfrost_gpu_soft_reset(pfdev); 409 410 panfrost_gpu_power_on(pfdev); 411 panfrost_mmu_reset(pfdev); 412 413 panfrost_jm_reset_interrupts(pfdev); 414 if (enable_job_int) 415 panfrost_jm_enable_interrupts(pfdev); 416 } 417 418 static int panfrost_device_runtime_resume(struct device *dev) 419 { 420 struct panfrost_device *pfdev = dev_get_drvdata(dev); 421 int ret; 422 423 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) { 424 ret = reset_control_deassert(pfdev->rstc); 425 if (ret) 426 return ret; 427 428 ret = clk_enable(pfdev->clock); 429 if (ret) 430 goto err_clk; 431 432 if (pfdev->bus_clock) { 433 ret = clk_enable(pfdev->bus_clock); 434 if (ret) 435 goto err_bus_clk; 436 } 437 } 438 439 panfrost_device_reset(pfdev, true); 440 panfrost_devfreq_resume(pfdev); 441 442 return 0; 443 444 err_bus_clk: 445 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) 446 clk_disable(pfdev->clock); 447 err_clk: 448 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) 449 reset_control_assert(pfdev->rstc); 450 return ret; 451 } 452 453 static int panfrost_device_runtime_suspend(struct device *dev) 454 { 455 struct panfrost_device *pfdev = dev_get_drvdata(dev); 456 457 if (!panfrost_jm_is_idle(pfdev)) 458 return -EBUSY; 459 460 panfrost_devfreq_suspend(pfdev); 461 panfrost_jm_suspend_irq(pfdev); 462 panfrost_mmu_suspend_irq(pfdev); 463 panfrost_gpu_suspend_irq(pfdev); 464 panfrost_gpu_power_off(pfdev); 465 466 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) { 467 if (pfdev->bus_clock) 468 clk_disable(pfdev->bus_clock); 469 470 clk_disable(pfdev->clock); 471 reset_control_assert(pfdev->rstc); 472 } 473 474 return 0; 475 } 476 477 static int panfrost_device_resume(struct device *dev) 478 { 479 struct panfrost_device *pfdev = dev_get_drvdata(dev); 480 int ret; 481 482 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) { 483 unsigned long freq = pfdev->pfdevfreq.fast_rate; 484 struct dev_pm_opp *opp; 485 486 opp = dev_pm_opp_find_freq_ceil(dev, &freq); 487 if (IS_ERR(opp)) 488 return PTR_ERR(opp); 489 dev_pm_opp_set_opp(dev, opp); 490 dev_pm_opp_put(opp); 491 } 492 493 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) { 494 ret = clk_enable(pfdev->clock); 495 if (ret) 496 goto err_clk; 497 498 if (pfdev->bus_clock) { 499 ret = clk_enable(pfdev->bus_clock); 500 if (ret) 501 goto err_bus_clk; 502 } 503 } 504 505 ret = pm_runtime_force_resume(dev); 506 if (ret) 507 goto err_resume; 508 509 return 0; 510 511 err_resume: 512 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock) 513 clk_disable(pfdev->bus_clock); 514 err_bus_clk: 515 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) 516 clk_disable(pfdev->clock); 517 err_clk: 518 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) 519 dev_pm_opp_set_opp(dev, NULL); 520 return ret; 521 } 522 523 static int panfrost_device_suspend(struct device *dev) 524 { 525 struct panfrost_device *pfdev = dev_get_drvdata(dev); 526 int ret; 527 528 ret = pm_runtime_force_suspend(dev); 529 if (ret) 530 return ret; 531 532 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) { 533 if (pfdev->bus_clock) 534 clk_disable(pfdev->bus_clock); 535 536 clk_disable(pfdev->clock); 537 } 538 539 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) 540 dev_pm_opp_set_opp(dev, NULL); 541 542 return 0; 543 } 544 545 EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = { 546 RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL) 547 SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume) 548 }; 549