1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 4 /* Copyright 2023 Collabora ltd. */ 5 6 #include <linux/clk.h> 7 #include <linux/mm.h> 8 #include <linux/platform_device.h> 9 #include <linux/pm_domain.h> 10 #include <linux/pm_runtime.h> 11 #include <linux/regulator/consumer.h> 12 #include <linux/reset.h> 13 14 #include <drm/drm_drv.h> 15 #include <drm/drm_managed.h> 16 17 #include "panthor_devfreq.h" 18 #include "panthor_device.h" 19 #include "panthor_fw.h" 20 #include "panthor_gpu.h" 21 #include "panthor_mmu.h" 22 #include "panthor_regs.h" 23 #include "panthor_sched.h" 24 25 static int panthor_clk_init(struct panthor_device *ptdev) 26 { 27 ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL); 28 if (IS_ERR(ptdev->clks.core)) 29 return dev_err_probe(ptdev->base.dev, 30 PTR_ERR(ptdev->clks.core), 31 "get 'core' clock failed"); 32 33 ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks"); 34 if (IS_ERR(ptdev->clks.stacks)) 35 return dev_err_probe(ptdev->base.dev, 36 PTR_ERR(ptdev->clks.stacks), 37 "get 'stacks' clock failed"); 38 39 ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup"); 40 if (IS_ERR(ptdev->clks.coregroup)) 41 return dev_err_probe(ptdev->base.dev, 42 PTR_ERR(ptdev->clks.coregroup), 43 "get 'coregroup' clock failed"); 44 45 drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core)); 46 return 0; 47 } 48 49 void panthor_device_unplug(struct panthor_device *ptdev) 50 { 51 /* This function can be called from two different path: the reset work 52 * and the platform device remove callback. drm_dev_unplug() doesn't 53 * deal with concurrent callers, so we have to protect drm_dev_unplug() 54 * calls with our own lock, and bail out if the device is already 55 * unplugged. 56 */ 57 mutex_lock(&ptdev->unplug.lock); 58 if (drm_dev_is_unplugged(&ptdev->base)) { 59 /* Someone beat us, release the lock and wait for the unplug 60 * operation to be reported as done. 61 **/ 62 mutex_unlock(&ptdev->unplug.lock); 63 wait_for_completion(&ptdev->unplug.done); 64 return; 65 } 66 67 /* Call drm_dev_unplug() so any access to HW blocks happening after 68 * that point get rejected. 69 */ 70 drm_dev_unplug(&ptdev->base); 71 72 /* We do the rest of the unplug with the unplug lock released, 73 * future callers will wait on ptdev->unplug.done anyway. 74 */ 75 mutex_unlock(&ptdev->unplug.lock); 76 77 drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0); 78 79 /* Now, try to cleanly shutdown the GPU before the device resources 80 * get reclaimed. 81 */ 82 panthor_sched_unplug(ptdev); 83 panthor_fw_unplug(ptdev); 84 panthor_mmu_unplug(ptdev); 85 panthor_gpu_unplug(ptdev); 86 87 pm_runtime_dont_use_autosuspend(ptdev->base.dev); 88 pm_runtime_put_sync_suspend(ptdev->base.dev); 89 90 /* If PM is disabled, we need to call the suspend handler manually. */ 91 if (!IS_ENABLED(CONFIG_PM)) 92 panthor_device_suspend(ptdev->base.dev); 93 94 /* Report the unplug operation as done to unblock concurrent 95 * panthor_device_unplug() callers. 96 */ 97 complete_all(&ptdev->unplug.done); 98 } 99 100 static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data) 101 { 102 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); 103 104 cancel_work_sync(&ptdev->reset.work); 105 destroy_workqueue(ptdev->reset.wq); 106 } 107 108 static void panthor_device_reset_work(struct work_struct *work) 109 { 110 struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work); 111 int ret = 0, cookie; 112 113 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) { 114 /* 115 * No need for a reset as the device has been (or will be) 116 * powered down 117 */ 118 atomic_set(&ptdev->reset.pending, 0); 119 return; 120 } 121 122 if (!drm_dev_enter(&ptdev->base, &cookie)) 123 return; 124 125 panthor_sched_pre_reset(ptdev); 126 panthor_fw_pre_reset(ptdev, true); 127 panthor_mmu_pre_reset(ptdev); 128 panthor_gpu_soft_reset(ptdev); 129 panthor_gpu_l2_power_on(ptdev); 130 panthor_mmu_post_reset(ptdev); 131 ret = panthor_fw_post_reset(ptdev); 132 atomic_set(&ptdev->reset.pending, 0); 133 panthor_sched_post_reset(ptdev, ret != 0); 134 drm_dev_exit(cookie); 135 136 if (ret) { 137 panthor_device_unplug(ptdev); 138 drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable."); 139 } 140 } 141 142 static bool panthor_device_is_initialized(struct panthor_device *ptdev) 143 { 144 return !!ptdev->scheduler; 145 } 146 147 static void panthor_device_free_page(struct drm_device *ddev, void *data) 148 { 149 __free_page(data); 150 } 151 152 int panthor_device_init(struct panthor_device *ptdev) 153 { 154 u32 *dummy_page_virt; 155 struct resource *res; 156 struct page *p; 157 int ret; 158 159 ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT; 160 161 init_completion(&ptdev->unplug.done); 162 ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock); 163 if (ret) 164 return ret; 165 166 ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock); 167 if (ret) 168 return ret; 169 170 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); 171 p = alloc_page(GFP_KERNEL | __GFP_ZERO); 172 if (!p) 173 return -ENOMEM; 174 175 ptdev->pm.dummy_latest_flush = p; 176 dummy_page_virt = page_address(p); 177 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page, 178 ptdev->pm.dummy_latest_flush); 179 if (ret) 180 return ret; 181 182 /* 183 * Set the dummy page holding the latest flush to 1. This will cause the 184 * flush to avoided as we know it isn't necessary if the submission 185 * happens while the dummy page is mapped. Zero cannot be used because 186 * that means 'always flush'. 187 */ 188 *dummy_page_virt = 1; 189 190 INIT_WORK(&ptdev->reset.work, panthor_device_reset_work); 191 ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0); 192 if (!ptdev->reset.wq) 193 return -ENOMEM; 194 195 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL); 196 if (ret) 197 return ret; 198 199 ret = panthor_clk_init(ptdev); 200 if (ret) 201 return ret; 202 203 ret = panthor_devfreq_init(ptdev); 204 if (ret) 205 return ret; 206 207 ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev), 208 0, &res); 209 if (IS_ERR(ptdev->iomem)) 210 return PTR_ERR(ptdev->iomem); 211 212 ptdev->phys_addr = res->start; 213 214 ret = devm_pm_runtime_enable(ptdev->base.dev); 215 if (ret) 216 return ret; 217 218 ret = pm_runtime_resume_and_get(ptdev->base.dev); 219 if (ret) 220 return ret; 221 222 /* If PM is disabled, we need to call panthor_device_resume() manually. */ 223 if (!IS_ENABLED(CONFIG_PM)) { 224 ret = panthor_device_resume(ptdev->base.dev); 225 if (ret) 226 return ret; 227 } 228 229 ret = panthor_gpu_init(ptdev); 230 if (ret) 231 goto err_rpm_put; 232 233 ret = panthor_mmu_init(ptdev); 234 if (ret) 235 goto err_unplug_gpu; 236 237 ret = panthor_fw_init(ptdev); 238 if (ret) 239 goto err_unplug_mmu; 240 241 ret = panthor_sched_init(ptdev); 242 if (ret) 243 goto err_unplug_fw; 244 245 /* ~3 frames */ 246 pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50); 247 pm_runtime_use_autosuspend(ptdev->base.dev); 248 249 ret = drm_dev_register(&ptdev->base, 0); 250 if (ret) 251 goto err_disable_autosuspend; 252 253 pm_runtime_put_autosuspend(ptdev->base.dev); 254 return 0; 255 256 err_disable_autosuspend: 257 pm_runtime_dont_use_autosuspend(ptdev->base.dev); 258 panthor_sched_unplug(ptdev); 259 260 err_unplug_fw: 261 panthor_fw_unplug(ptdev); 262 263 err_unplug_mmu: 264 panthor_mmu_unplug(ptdev); 265 266 err_unplug_gpu: 267 panthor_gpu_unplug(ptdev); 268 269 err_rpm_put: 270 pm_runtime_put_sync_suspend(ptdev->base.dev); 271 return ret; 272 } 273 274 #define PANTHOR_EXCEPTION(id) \ 275 [DRM_PANTHOR_EXCEPTION_ ## id] = { \ 276 .name = #id, \ 277 } 278 279 struct panthor_exception_info { 280 const char *name; 281 }; 282 283 static const struct panthor_exception_info panthor_exception_infos[] = { 284 PANTHOR_EXCEPTION(OK), 285 PANTHOR_EXCEPTION(TERMINATED), 286 PANTHOR_EXCEPTION(KABOOM), 287 PANTHOR_EXCEPTION(EUREKA), 288 PANTHOR_EXCEPTION(ACTIVE), 289 PANTHOR_EXCEPTION(CS_RES_TERM), 290 PANTHOR_EXCEPTION(CS_CONFIG_FAULT), 291 PANTHOR_EXCEPTION(CS_UNRECOVERABLE), 292 PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT), 293 PANTHOR_EXCEPTION(CS_BUS_FAULT), 294 PANTHOR_EXCEPTION(CS_INSTR_INVALID), 295 PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW), 296 PANTHOR_EXCEPTION(CS_INHERIT_FAULT), 297 PANTHOR_EXCEPTION(INSTR_INVALID_PC), 298 PANTHOR_EXCEPTION(INSTR_INVALID_ENC), 299 PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT), 300 PANTHOR_EXCEPTION(DATA_INVALID_FAULT), 301 PANTHOR_EXCEPTION(TILE_RANGE_FAULT), 302 PANTHOR_EXCEPTION(ADDR_RANGE_FAULT), 303 PANTHOR_EXCEPTION(IMPRECISE_FAULT), 304 PANTHOR_EXCEPTION(OOM), 305 PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR), 306 PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT), 307 PANTHOR_EXCEPTION(GPU_BUS_FAULT), 308 PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT), 309 PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT), 310 PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT), 311 PANTHOR_EXCEPTION(TRANSLATION_FAULT_0), 312 PANTHOR_EXCEPTION(TRANSLATION_FAULT_1), 313 PANTHOR_EXCEPTION(TRANSLATION_FAULT_2), 314 PANTHOR_EXCEPTION(TRANSLATION_FAULT_3), 315 PANTHOR_EXCEPTION(TRANSLATION_FAULT_4), 316 PANTHOR_EXCEPTION(PERM_FAULT_0), 317 PANTHOR_EXCEPTION(PERM_FAULT_1), 318 PANTHOR_EXCEPTION(PERM_FAULT_2), 319 PANTHOR_EXCEPTION(PERM_FAULT_3), 320 PANTHOR_EXCEPTION(ACCESS_FLAG_1), 321 PANTHOR_EXCEPTION(ACCESS_FLAG_2), 322 PANTHOR_EXCEPTION(ACCESS_FLAG_3), 323 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN), 324 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0), 325 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1), 326 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2), 327 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3), 328 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0), 329 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1), 330 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2), 331 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3), 332 }; 333 334 const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code) 335 { 336 if (exception_code >= ARRAY_SIZE(panthor_exception_infos) || 337 !panthor_exception_infos[exception_code].name) 338 return "Unknown exception type"; 339 340 return panthor_exception_infos[exception_code].name; 341 } 342 343 static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf) 344 { 345 struct vm_area_struct *vma = vmf->vma; 346 struct panthor_device *ptdev = vma->vm_private_data; 347 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT; 348 unsigned long pfn; 349 pgprot_t pgprot; 350 vm_fault_t ret; 351 bool active; 352 int cookie; 353 354 if (!drm_dev_enter(&ptdev->base, &cookie)) 355 return VM_FAULT_SIGBUS; 356 357 mutex_lock(&ptdev->pm.mmio_lock); 358 active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE; 359 360 switch (offset) { 361 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: 362 if (active) 363 pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID); 364 else 365 pfn = page_to_pfn(ptdev->pm.dummy_latest_flush); 366 break; 367 368 default: 369 ret = VM_FAULT_SIGBUS; 370 goto out_unlock; 371 } 372 373 pgprot = vma->vm_page_prot; 374 if (active) 375 pgprot = pgprot_noncached(pgprot); 376 377 ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot); 378 379 out_unlock: 380 mutex_unlock(&ptdev->pm.mmio_lock); 381 drm_dev_exit(cookie); 382 return ret; 383 } 384 385 static const struct vm_operations_struct panthor_mmio_vm_ops = { 386 .fault = panthor_mmio_vm_fault, 387 }; 388 389 int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma) 390 { 391 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT; 392 393 if ((vma->vm_flags & VM_SHARED) == 0) 394 return -EINVAL; 395 396 switch (offset) { 397 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: 398 if (vma->vm_end - vma->vm_start != PAGE_SIZE || 399 (vma->vm_flags & (VM_WRITE | VM_EXEC))) 400 return -EINVAL; 401 vm_flags_clear(vma, VM_MAYWRITE); 402 403 break; 404 405 default: 406 return -EINVAL; 407 } 408 409 /* Defer actual mapping to the fault handler. */ 410 vma->vm_private_data = ptdev; 411 vma->vm_ops = &panthor_mmio_vm_ops; 412 vm_flags_set(vma, 413 VM_IO | VM_DONTCOPY | VM_DONTEXPAND | 414 VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP); 415 return 0; 416 } 417 418 int panthor_device_resume(struct device *dev) 419 { 420 struct panthor_device *ptdev = dev_get_drvdata(dev); 421 int ret, cookie; 422 423 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED) 424 return -EINVAL; 425 426 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING); 427 428 ret = clk_prepare_enable(ptdev->clks.core); 429 if (ret) 430 goto err_set_suspended; 431 432 ret = clk_prepare_enable(ptdev->clks.stacks); 433 if (ret) 434 goto err_disable_core_clk; 435 436 ret = clk_prepare_enable(ptdev->clks.coregroup); 437 if (ret) 438 goto err_disable_stacks_clk; 439 440 ret = panthor_devfreq_resume(ptdev); 441 if (ret) 442 goto err_disable_coregroup_clk; 443 444 if (panthor_device_is_initialized(ptdev) && 445 drm_dev_enter(&ptdev->base, &cookie)) { 446 panthor_gpu_resume(ptdev); 447 panthor_mmu_resume(ptdev); 448 ret = drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev)); 449 if (!ret) { 450 panthor_sched_resume(ptdev); 451 } else { 452 panthor_mmu_suspend(ptdev); 453 panthor_gpu_suspend(ptdev); 454 } 455 456 drm_dev_exit(cookie); 457 458 if (ret) 459 goto err_suspend_devfreq; 460 } 461 462 if (atomic_read(&ptdev->reset.pending)) 463 queue_work(ptdev->reset.wq, &ptdev->reset.work); 464 465 /* Clear all IOMEM mappings pointing to this device after we've 466 * resumed. This way the fake mappings pointing to the dummy pages 467 * are removed and the real iomem mapping will be restored on next 468 * access. 469 */ 470 mutex_lock(&ptdev->pm.mmio_lock); 471 unmap_mapping_range(ptdev->base.anon_inode->i_mapping, 472 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); 473 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE); 474 mutex_unlock(&ptdev->pm.mmio_lock); 475 return 0; 476 477 err_suspend_devfreq: 478 panthor_devfreq_suspend(ptdev); 479 480 err_disable_coregroup_clk: 481 clk_disable_unprepare(ptdev->clks.coregroup); 482 483 err_disable_stacks_clk: 484 clk_disable_unprepare(ptdev->clks.stacks); 485 486 err_disable_core_clk: 487 clk_disable_unprepare(ptdev->clks.core); 488 489 err_set_suspended: 490 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); 491 return ret; 492 } 493 494 int panthor_device_suspend(struct device *dev) 495 { 496 struct panthor_device *ptdev = dev_get_drvdata(dev); 497 int ret, cookie; 498 499 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) 500 return -EINVAL; 501 502 /* Clear all IOMEM mappings pointing to this device before we 503 * shutdown the power-domain and clocks. Failing to do that results 504 * in external aborts when the process accesses the iomem region. 505 * We change the state and call unmap_mapping_range() with the 506 * mmio_lock held to make sure the vm_fault handler won't set up 507 * invalid mappings. 508 */ 509 mutex_lock(&ptdev->pm.mmio_lock); 510 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING); 511 unmap_mapping_range(ptdev->base.anon_inode->i_mapping, 512 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); 513 mutex_unlock(&ptdev->pm.mmio_lock); 514 515 if (panthor_device_is_initialized(ptdev) && 516 drm_dev_enter(&ptdev->base, &cookie)) { 517 cancel_work_sync(&ptdev->reset.work); 518 519 /* We prepare everything as if we were resetting the GPU. 520 * The end of the reset will happen in the resume path though. 521 */ 522 panthor_sched_suspend(ptdev); 523 panthor_fw_suspend(ptdev); 524 panthor_mmu_suspend(ptdev); 525 panthor_gpu_suspend(ptdev); 526 drm_dev_exit(cookie); 527 } 528 529 ret = panthor_devfreq_suspend(ptdev); 530 if (ret) { 531 if (panthor_device_is_initialized(ptdev) && 532 drm_dev_enter(&ptdev->base, &cookie)) { 533 panthor_gpu_resume(ptdev); 534 panthor_mmu_resume(ptdev); 535 drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev)); 536 panthor_sched_resume(ptdev); 537 drm_dev_exit(cookie); 538 } 539 540 goto err_set_active; 541 } 542 543 clk_disable_unprepare(ptdev->clks.coregroup); 544 clk_disable_unprepare(ptdev->clks.stacks); 545 clk_disable_unprepare(ptdev->clks.core); 546 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); 547 return 0; 548 549 err_set_active: 550 /* If something failed and we have to revert back to an 551 * active state, we also need to clear the MMIO userspace 552 * mappings, so any dumb pages that were mapped while we 553 * were trying to suspend gets invalidated. 554 */ 555 mutex_lock(&ptdev->pm.mmio_lock); 556 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE); 557 unmap_mapping_range(ptdev->base.anon_inode->i_mapping, 558 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); 559 mutex_unlock(&ptdev->pm.mmio_lock); 560 return ret; 561 } 562