1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 4 5 #include <linux/clk.h> 6 #include <linux/reset.h> 7 #include <linux/platform_device.h> 8 #include <linux/pm_domain.h> 9 #include <linux/pm_runtime.h> 10 #include <linux/regulator/consumer.h> 11 12 #include "panfrost_device.h" 13 #include "panfrost_devfreq.h" 14 #include "panfrost_features.h" 15 #include "panfrost_issues.h" 16 #include "panfrost_gpu.h" 17 #include "panfrost_job.h" 18 #include "panfrost_mmu.h" 19 #include "panfrost_perfcnt.h" 20 21 static int panfrost_reset_init(struct panfrost_device *pfdev) 22 { 23 pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->base.dev); 24 if (IS_ERR(pfdev->rstc)) { 25 dev_err(pfdev->base.dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc)); 26 return PTR_ERR(pfdev->rstc); 27 } 28 29 return reset_control_deassert(pfdev->rstc); 30 } 31 32 static void panfrost_reset_fini(struct panfrost_device *pfdev) 33 { 34 reset_control_assert(pfdev->rstc); 35 } 36 37 static int panfrost_clk_init(struct panfrost_device *pfdev) 38 { 39 int err; 40 unsigned long rate; 41 42 pfdev->clock = devm_clk_get(pfdev->base.dev, NULL); 43 if (IS_ERR(pfdev->clock)) { 44 dev_err(pfdev->base.dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock)); 45 return PTR_ERR(pfdev->clock); 46 } 47 48 rate = clk_get_rate(pfdev->clock); 49 dev_info(pfdev->base.dev, "clock rate = %lu\n", rate); 50 51 err = clk_prepare_enable(pfdev->clock); 52 if (err) 53 return err; 54 55 pfdev->bus_clock = devm_clk_get_optional(pfdev->base.dev, "bus"); 56 if (IS_ERR(pfdev->bus_clock)) { 57 dev_err(pfdev->base.dev, "get bus_clock failed %ld\n", 58 PTR_ERR(pfdev->bus_clock)); 59 err = PTR_ERR(pfdev->bus_clock); 60 goto disable_clock; 61 } 62 63 if (pfdev->bus_clock) { 64 rate = clk_get_rate(pfdev->bus_clock); 65 dev_info(pfdev->base.dev, "bus_clock rate = %lu\n", rate); 66 67 err = clk_prepare_enable(pfdev->bus_clock); 68 if (err) 69 goto disable_clock; 70 } 71 72 return 0; 73 74 disable_clock: 75 clk_disable_unprepare(pfdev->clock); 76 77 return err; 78 } 79 80 static void panfrost_clk_fini(struct panfrost_device *pfdev) 81 { 82 clk_disable_unprepare(pfdev->bus_clock); 83 clk_disable_unprepare(pfdev->clock); 84 } 85 86 static int panfrost_regulator_init(struct panfrost_device *pfdev) 87 { 88 int ret, i; 89 90 pfdev->regulators = devm_kcalloc(pfdev->base.dev, pfdev->comp->num_supplies, 91 sizeof(*pfdev->regulators), 92 GFP_KERNEL); 93 if (!pfdev->regulators) 94 return -ENOMEM; 95 96 for (i = 0; i < pfdev->comp->num_supplies; i++) 97 pfdev->regulators[i].supply = pfdev->comp->supply_names[i]; 98 99 ret = devm_regulator_bulk_get(pfdev->base.dev, 100 pfdev->comp->num_supplies, 101 pfdev->regulators); 102 if (ret < 0) { 103 if (ret != -EPROBE_DEFER) 104 dev_err(pfdev->base.dev, "failed to get regulators: %d\n", 105 ret); 106 return ret; 107 } 108 109 ret = regulator_bulk_enable(pfdev->comp->num_supplies, 110 pfdev->regulators); 111 if (ret < 0) { 112 dev_err(pfdev->base.dev, "failed to enable regulators: %d\n", ret); 113 return ret; 114 } 115 116 return 0; 117 } 118 119 static void panfrost_regulator_fini(struct panfrost_device *pfdev) 120 { 121 if (!pfdev->regulators) 122 return; 123 124 regulator_bulk_disable(pfdev->comp->num_supplies, pfdev->regulators); 125 } 126 127 static void panfrost_pm_domain_fini(struct panfrost_device *pfdev) 128 { 129 int i; 130 131 for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) { 132 if (!pfdev->pm_domain_devs[i]) 133 break; 134 135 if (pfdev->pm_domain_links[i]) 136 device_link_del(pfdev->pm_domain_links[i]); 137 138 dev_pm_domain_detach(pfdev->pm_domain_devs[i], true); 139 } 140 } 141 142 static int panfrost_pm_domain_init(struct panfrost_device *pfdev) 143 { 144 int err; 145 int i, num_domains; 146 147 num_domains = of_count_phandle_with_args(pfdev->base.dev->of_node, 148 "power-domains", 149 "#power-domain-cells"); 150 151 /* 152 * Single domain is handled by the core, and, if only a single power 153 * the power domain is requested, the property is optional. 154 */ 155 if (num_domains < 2 && pfdev->comp->num_pm_domains < 2) 156 return 0; 157 158 if (num_domains != pfdev->comp->num_pm_domains) { 159 dev_err(pfdev->base.dev, 160 "Incorrect number of power domains: %d provided, %d needed\n", 161 num_domains, pfdev->comp->num_pm_domains); 162 return -EINVAL; 163 } 164 165 if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs), 166 "Too many supplies in compatible structure.\n")) 167 return -EINVAL; 168 169 for (i = 0; i < num_domains; i++) { 170 pfdev->pm_domain_devs[i] = 171 dev_pm_domain_attach_by_name(pfdev->base.dev, 172 pfdev->comp->pm_domain_names[i]); 173 if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) { 174 err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA; 175 pfdev->pm_domain_devs[i] = NULL; 176 dev_err(pfdev->base.dev, 177 "failed to get pm-domain %s(%d): %d\n", 178 pfdev->comp->pm_domain_names[i], i, err); 179 goto err; 180 } 181 182 pfdev->pm_domain_links[i] = 183 device_link_add(pfdev->base.dev, 184 pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME | 185 DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE); 186 if (!pfdev->pm_domain_links[i]) { 187 dev_err(pfdev->pm_domain_devs[i], 188 "adding device link failed!\n"); 189 err = -ENODEV; 190 goto err; 191 } 192 } 193 194 return 0; 195 196 err: 197 panfrost_pm_domain_fini(pfdev); 198 return err; 199 } 200 201 int panfrost_device_init(struct panfrost_device *pfdev) 202 { 203 int err; 204 205 mutex_init(&pfdev->sched_lock); 206 INIT_LIST_HEAD(&pfdev->scheduled_jobs); 207 INIT_LIST_HEAD(&pfdev->as_lru_list); 208 209 spin_lock_init(&pfdev->as_lock); 210 211 spin_lock_init(&pfdev->cycle_counter.lock); 212 213 #ifdef CONFIG_DEBUG_FS 214 mutex_init(&pfdev->debugfs.gems_lock); 215 INIT_LIST_HEAD(&pfdev->debugfs.gems_list); 216 #endif 217 218 err = panfrost_pm_domain_init(pfdev); 219 if (err) 220 return err; 221 222 err = panfrost_reset_init(pfdev); 223 if (err) { 224 dev_err(pfdev->base.dev, "reset init failed %d\n", err); 225 goto out_pm_domain; 226 } 227 228 err = panfrost_clk_init(pfdev); 229 if (err) { 230 dev_err(pfdev->base.dev, "clk init failed %d\n", err); 231 goto out_reset; 232 } 233 234 err = panfrost_devfreq_init(pfdev); 235 if (err) { 236 if (err != -EPROBE_DEFER) 237 dev_err(pfdev->base.dev, "devfreq init failed %d\n", err); 238 goto out_clk; 239 } 240 241 /* OPP will handle regulators */ 242 if (!pfdev->pfdevfreq.opp_of_table_added) { 243 err = panfrost_regulator_init(pfdev); 244 if (err) 245 goto out_devfreq; 246 } 247 248 pfdev->iomem = devm_platform_ioremap_resource(to_platform_device(pfdev->base.dev), 0); 249 if (IS_ERR(pfdev->iomem)) { 250 err = PTR_ERR(pfdev->iomem); 251 goto out_regulator; 252 } 253 254 err = panfrost_gpu_init(pfdev); 255 if (err) 256 goto out_regulator; 257 258 err = panfrost_mmu_init(pfdev); 259 if (err) 260 goto out_gpu; 261 262 err = panfrost_jm_init(pfdev); 263 if (err) 264 goto out_mmu; 265 266 err = panfrost_perfcnt_init(pfdev); 267 if (err) 268 goto out_job; 269 270 return 0; 271 out_job: 272 panfrost_jm_fini(pfdev); 273 out_mmu: 274 panfrost_mmu_fini(pfdev); 275 out_gpu: 276 panfrost_gpu_fini(pfdev); 277 out_regulator: 278 panfrost_regulator_fini(pfdev); 279 out_devfreq: 280 panfrost_devfreq_fini(pfdev); 281 out_clk: 282 panfrost_clk_fini(pfdev); 283 out_reset: 284 panfrost_reset_fini(pfdev); 285 out_pm_domain: 286 panfrost_pm_domain_fini(pfdev); 287 return err; 288 } 289 290 void panfrost_device_fini(struct panfrost_device *pfdev) 291 { 292 panfrost_perfcnt_fini(pfdev); 293 panfrost_jm_fini(pfdev); 294 panfrost_mmu_fini(pfdev); 295 panfrost_gpu_fini(pfdev); 296 panfrost_devfreq_fini(pfdev); 297 panfrost_regulator_fini(pfdev); 298 panfrost_clk_fini(pfdev); 299 panfrost_reset_fini(pfdev); 300 panfrost_pm_domain_fini(pfdev); 301 } 302 303 #define PANFROST_EXCEPTION(id) \ 304 [DRM_PANFROST_EXCEPTION_ ## id] = { \ 305 .name = #id, \ 306 } 307 308 struct panfrost_exception_info { 309 const char *name; 310 }; 311 312 static const struct panfrost_exception_info panfrost_exception_infos[] = { 313 PANFROST_EXCEPTION(OK), 314 PANFROST_EXCEPTION(DONE), 315 PANFROST_EXCEPTION(INTERRUPTED), 316 PANFROST_EXCEPTION(STOPPED), 317 PANFROST_EXCEPTION(TERMINATED), 318 PANFROST_EXCEPTION(KABOOM), 319 PANFROST_EXCEPTION(EUREKA), 320 PANFROST_EXCEPTION(ACTIVE), 321 PANFROST_EXCEPTION(JOB_CONFIG_FAULT), 322 PANFROST_EXCEPTION(JOB_POWER_FAULT), 323 PANFROST_EXCEPTION(JOB_READ_FAULT), 324 PANFROST_EXCEPTION(JOB_WRITE_FAULT), 325 PANFROST_EXCEPTION(JOB_AFFINITY_FAULT), 326 PANFROST_EXCEPTION(JOB_BUS_FAULT), 327 PANFROST_EXCEPTION(INSTR_INVALID_PC), 328 PANFROST_EXCEPTION(INSTR_INVALID_ENC), 329 PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH), 330 PANFROST_EXCEPTION(INSTR_OPERAND_FAULT), 331 PANFROST_EXCEPTION(INSTR_TLS_FAULT), 332 PANFROST_EXCEPTION(INSTR_BARRIER_FAULT), 333 PANFROST_EXCEPTION(INSTR_ALIGN_FAULT), 334 PANFROST_EXCEPTION(DATA_INVALID_FAULT), 335 PANFROST_EXCEPTION(TILE_RANGE_FAULT), 336 PANFROST_EXCEPTION(ADDR_RANGE_FAULT), 337 PANFROST_EXCEPTION(IMPRECISE_FAULT), 338 PANFROST_EXCEPTION(OOM), 339 PANFROST_EXCEPTION(OOM_AFBC), 340 PANFROST_EXCEPTION(UNKNOWN), 341 PANFROST_EXCEPTION(DELAYED_BUS_FAULT), 342 PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT), 343 PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT), 344 PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT), 345 PANFROST_EXCEPTION(TRANSLATION_FAULT_0), 346 PANFROST_EXCEPTION(TRANSLATION_FAULT_1), 347 PANFROST_EXCEPTION(TRANSLATION_FAULT_2), 348 PANFROST_EXCEPTION(TRANSLATION_FAULT_3), 349 PANFROST_EXCEPTION(TRANSLATION_FAULT_4), 350 PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY), 351 PANFROST_EXCEPTION(PERM_FAULT_0), 352 PANFROST_EXCEPTION(PERM_FAULT_1), 353 PANFROST_EXCEPTION(PERM_FAULT_2), 354 PANFROST_EXCEPTION(PERM_FAULT_3), 355 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0), 356 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1), 357 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2), 358 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3), 359 PANFROST_EXCEPTION(ACCESS_FLAG_0), 360 PANFROST_EXCEPTION(ACCESS_FLAG_1), 361 PANFROST_EXCEPTION(ACCESS_FLAG_2), 362 PANFROST_EXCEPTION(ACCESS_FLAG_3), 363 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0), 364 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1), 365 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2), 366 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3), 367 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0), 368 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1), 369 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2), 370 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3), 371 PANFROST_EXCEPTION(MEM_ATTR_FAULT_0), 372 PANFROST_EXCEPTION(MEM_ATTR_FAULT_1), 373 PANFROST_EXCEPTION(MEM_ATTR_FAULT_2), 374 PANFROST_EXCEPTION(MEM_ATTR_FAULT_3), 375 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0), 376 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1), 377 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2), 378 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3), 379 }; 380 381 const char *panfrost_exception_name(u32 exception_code) 382 { 383 if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) || 384 !panfrost_exception_infos[exception_code].name)) 385 return "Unknown exception type"; 386 387 return panfrost_exception_infos[exception_code].name; 388 } 389 390 bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev, 391 u32 exception_code) 392 { 393 /* If an occlusion query write causes a bus fault on affected GPUs, 394 * future fragment jobs may hang. Reset to workaround. 395 */ 396 if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT) 397 return panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_3076); 398 399 /* No other GPUs we support need a reset */ 400 return false; 401 } 402 403 void panfrost_device_reset(struct panfrost_device *pfdev, bool enable_job_int) 404 { 405 panfrost_gpu_soft_reset(pfdev); 406 407 panfrost_gpu_power_on(pfdev); 408 panfrost_mmu_reset(pfdev); 409 410 panfrost_jm_reset_interrupts(pfdev); 411 if (enable_job_int) 412 panfrost_jm_enable_interrupts(pfdev); 413 } 414 415 static int panfrost_device_runtime_resume(struct device *dev) 416 { 417 struct panfrost_device *pfdev = dev_get_drvdata(dev); 418 int ret; 419 420 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) { 421 ret = reset_control_deassert(pfdev->rstc); 422 if (ret) 423 return ret; 424 425 ret = clk_enable(pfdev->clock); 426 if (ret) 427 goto err_clk; 428 429 if (pfdev->bus_clock) { 430 ret = clk_enable(pfdev->bus_clock); 431 if (ret) 432 goto err_bus_clk; 433 } 434 } 435 436 panfrost_device_reset(pfdev, true); 437 panfrost_devfreq_resume(pfdev); 438 439 return 0; 440 441 err_bus_clk: 442 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) 443 clk_disable(pfdev->clock); 444 err_clk: 445 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) 446 reset_control_assert(pfdev->rstc); 447 return ret; 448 } 449 450 static int panfrost_device_runtime_suspend(struct device *dev) 451 { 452 struct panfrost_device *pfdev = dev_get_drvdata(dev); 453 454 if (!panfrost_jm_is_idle(pfdev)) 455 return -EBUSY; 456 457 panfrost_devfreq_suspend(pfdev); 458 panfrost_jm_suspend_irq(pfdev); 459 panfrost_mmu_suspend_irq(pfdev); 460 panfrost_gpu_suspend_irq(pfdev); 461 panfrost_gpu_power_off(pfdev); 462 463 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) { 464 if (pfdev->bus_clock) 465 clk_disable(pfdev->bus_clock); 466 467 clk_disable(pfdev->clock); 468 reset_control_assert(pfdev->rstc); 469 } 470 471 return 0; 472 } 473 474 static int panfrost_device_resume(struct device *dev) 475 { 476 struct panfrost_device *pfdev = dev_get_drvdata(dev); 477 int ret; 478 479 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) { 480 unsigned long freq = pfdev->pfdevfreq.fast_rate; 481 struct dev_pm_opp *opp; 482 483 opp = dev_pm_opp_find_freq_ceil(dev, &freq); 484 if (IS_ERR(opp)) 485 return PTR_ERR(opp); 486 dev_pm_opp_set_opp(dev, opp); 487 dev_pm_opp_put(opp); 488 } 489 490 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) { 491 ret = clk_enable(pfdev->clock); 492 if (ret) 493 goto err_clk; 494 495 if (pfdev->bus_clock) { 496 ret = clk_enable(pfdev->bus_clock); 497 if (ret) 498 goto err_bus_clk; 499 } 500 } 501 502 ret = pm_runtime_force_resume(dev); 503 if (ret) 504 goto err_resume; 505 506 return 0; 507 508 err_resume: 509 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock) 510 clk_disable(pfdev->bus_clock); 511 err_bus_clk: 512 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) 513 clk_disable(pfdev->clock); 514 err_clk: 515 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) 516 dev_pm_opp_set_opp(dev, NULL); 517 return ret; 518 } 519 520 static int panfrost_device_suspend(struct device *dev) 521 { 522 struct panfrost_device *pfdev = dev_get_drvdata(dev); 523 int ret; 524 525 ret = pm_runtime_force_suspend(dev); 526 if (ret) 527 return ret; 528 529 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) { 530 if (pfdev->bus_clock) 531 clk_disable(pfdev->bus_clock); 532 533 clk_disable(pfdev->clock); 534 } 535 536 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) 537 dev_pm_opp_set_opp(dev, NULL); 538 539 return 0; 540 } 541 542 EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = { 543 RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL) 544 SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume) 545 }; 546