xref: /linux/drivers/gpu/drm/panthor/panthor_device.c (revision 72856afd33f2fa166c06f1536003e300e51ecf09) 
1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4 /* Copyright 2023 Collabora ltd. */
5 /* Copyright 2025 ARM Limited. All rights reserved. */
6 
7 #include <linux/clk.h>
8 #include <linux/mm.h>
9 #include <linux/platform_device.h>
10 #include <linux/pm_domain.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/regulator/consumer.h>
13 #include <linux/reset.h>
14 
15 #include <drm/drm_drv.h>
16 #include <drm/drm_managed.h>
17 #include <drm/drm_print.h>
18 
19 #include "panthor_devfreq.h"
20 #include "panthor_device.h"
21 #include "panthor_fw.h"
22 #include "panthor_fw_regs.h"
23 #include "panthor_gem.h"
24 #include "panthor_gpu.h"
25 #include "panthor_hw.h"
26 #include "panthor_mmu.h"
27 #include "panthor_pwr.h"
28 #include "panthor_sched.h"
29 
30 static int panthor_clk_init(struct panthor_device *ptdev)
31 {
32 	ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
33 	if (IS_ERR(ptdev->clks.core))
34 		return dev_err_probe(ptdev->base.dev,
35 				     PTR_ERR(ptdev->clks.core),
36 				     "get 'core' clock failed");
37 
38 	ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");
39 	if (IS_ERR(ptdev->clks.stacks))
40 		return dev_err_probe(ptdev->base.dev,
41 				     PTR_ERR(ptdev->clks.stacks),
42 				     "get 'stacks' clock failed");
43 
44 	ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");
45 	if (IS_ERR(ptdev->clks.coregroup))
46 		return dev_err_probe(ptdev->base.dev,
47 				     PTR_ERR(ptdev->clks.coregroup),
48 				     "get 'coregroup' clock failed");
49 
50 	drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
51 	return 0;
52 }
53 
54 static int panthor_init_power(struct device *dev)
55 {
56 	struct dev_pm_domain_list  *pd_list = NULL;
57 
58 	if (dev->pm_domain)
59 		return 0;
60 
61 	return devm_pm_domain_attach_list(dev, NULL, &pd_list);
62 }
63 
64 void panthor_device_unplug(struct panthor_device *ptdev)
65 {
66 	/* This function can be called from two different path: the reset work
67 	 * and the platform device remove callback. drm_dev_unplug() doesn't
68 	 * deal with concurrent callers, so we have to protect drm_dev_unplug()
69 	 * calls with our own lock, and bail out if the device is already
70 	 * unplugged.
71 	 */
72 	mutex_lock(&ptdev->unplug.lock);
73 	if (drm_dev_is_unplugged(&ptdev->base)) {
74 		/* Someone beat us, release the lock and wait for the unplug
75 		 * operation to be reported as done.
76 		 **/
77 		mutex_unlock(&ptdev->unplug.lock);
78 		wait_for_completion(&ptdev->unplug.done);
79 		return;
80 	}
81 
82 	drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);
83 
84 	/* Call drm_dev_unplug() so any access to HW blocks happening after
85 	 * that point get rejected.
86 	 */
87 	drm_dev_unplug(&ptdev->base);
88 
89 	/* We do the rest of the unplug with the unplug lock released,
90 	 * future callers will wait on ptdev->unplug.done anyway.
91 	 */
92 	mutex_unlock(&ptdev->unplug.lock);
93 
94 	/* Now, try to cleanly shutdown the GPU before the device resources
95 	 * get reclaimed.
96 	 */
97 	panthor_sched_unplug(ptdev);
98 	panthor_fw_unplug(ptdev);
99 	panthor_mmu_unplug(ptdev);
100 	panthor_gem_shrinker_unplug(ptdev);
101 	panthor_gpu_unplug(ptdev);
102 	panthor_pwr_unplug(ptdev);
103 
104 	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
105 	pm_runtime_put_sync_suspend(ptdev->base.dev);
106 
107 	/* If PM is disabled, we need to call the suspend handler manually. */
108 	if (!IS_ENABLED(CONFIG_PM))
109 		panthor_device_suspend(ptdev->base.dev);
110 
111 	/* Report the unplug operation as done to unblock concurrent
112 	 * panthor_device_unplug() callers.
113 	 */
114 	complete_all(&ptdev->unplug.done);
115 }
116 
117 static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
118 {
119 	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
120 
121 	disable_work_sync(&ptdev->reset.work);
122 	destroy_workqueue(ptdev->reset.wq);
123 }
124 
125 static void panthor_device_reset_work(struct work_struct *work)
126 {
127 	struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
128 	int ret = 0, cookie;
129 
130 	/* If the device is entering suspend, we don't reset. A slow reset will
131 	 * be forced at resume time instead.
132 	 */
133 	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
134 		return;
135 
136 	if (!drm_dev_enter(&ptdev->base, &cookie))
137 		return;
138 
139 	panthor_sched_pre_reset(ptdev);
140 	panthor_fw_pre_reset(ptdev, true);
141 	panthor_mmu_pre_reset(ptdev);
142 	panthor_hw_soft_reset(ptdev);
143 	panthor_hw_l2_power_on(ptdev);
144 	panthor_mmu_post_reset(ptdev);
145 	ret = panthor_fw_post_reset(ptdev);
146 	atomic_set(&ptdev->reset.pending, 0);
147 	panthor_sched_post_reset(ptdev, ret != 0);
148 	drm_dev_exit(cookie);
149 
150 	if (ret) {
151 		panthor_device_unplug(ptdev);
152 		drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
153 	}
154 }
155 
156 static bool panthor_device_is_initialized(struct panthor_device *ptdev)
157 {
158 	return !!ptdev->scheduler;
159 }
160 
161 static void panthor_device_free_page(struct drm_device *ddev, void *data)
162 {
163 	__free_page(data);
164 }
165 
166 int panthor_device_init(struct panthor_device *ptdev)
167 {
168 	u32 *dummy_page_virt;
169 	struct resource *res;
170 	struct page *p;
171 	int ret;
172 
173 	ptdev->soc_data = of_device_get_match_data(ptdev->base.dev);
174 
175 	init_completion(&ptdev->unplug.done);
176 	ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
177 	if (ret)
178 		return ret;
179 
180 	ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
181 	if (ret)
182 		return ret;
183 
184 #ifdef CONFIG_DEBUG_FS
185 	drmm_mutex_init(&ptdev->base, &ptdev->gems.lock);
186 	INIT_LIST_HEAD(&ptdev->gems.node);
187 #endif
188 
189 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
190 	p = alloc_page(GFP_KERNEL | __GFP_ZERO);
191 	if (!p)
192 		return -ENOMEM;
193 
194 	ptdev->pm.dummy_latest_flush = p;
195 	dummy_page_virt = page_address(p);
196 	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
197 				       ptdev->pm.dummy_latest_flush);
198 	if (ret)
199 		return ret;
200 
201 	/*
202 	 * Set the dummy page holding the latest flush to 1. This will cause the
203 	 * flush to avoided as we know it isn't necessary if the submission
204 	 * happens while the dummy page is mapped. Zero cannot be used because
205 	 * that means 'always flush'.
206 	 */
207 	*dummy_page_virt = 1;
208 
209 	INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
210 	disable_work(&ptdev->reset.work);
211 	ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
212 	if (!ptdev->reset.wq)
213 		return -ENOMEM;
214 
215 	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
216 	if (ret)
217 		return ret;
218 
219 	ret = panthor_clk_init(ptdev);
220 	if (ret)
221 		return ret;
222 
223 	ret = panthor_init_power(ptdev->base.dev);
224 	if (ret < 0) {
225 		drm_err(&ptdev->base, "init power domains failed, ret=%d", ret);
226 		return ret;
227 	}
228 
229 	ret = panthor_devfreq_init(ptdev);
230 	if (ret)
231 		return ret;
232 
233 	ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
234 							      0, &res);
235 	if (IS_ERR(ptdev->iomem))
236 		return PTR_ERR(ptdev->iomem);
237 
238 	ptdev->phys_addr = res->start;
239 
240 	ret = devm_pm_runtime_enable(ptdev->base.dev);
241 	if (ret)
242 		return ret;
243 
244 	ret = pm_runtime_resume_and_get(ptdev->base.dev);
245 	if (ret)
246 		return ret;
247 
248 	/* If PM is disabled, we need to call panthor_device_resume() manually. */
249 	if (!IS_ENABLED(CONFIG_PM)) {
250 		ret = panthor_device_resume(ptdev->base.dev);
251 		if (ret)
252 			return ret;
253 	}
254 
255 	ret = panthor_hw_init(ptdev);
256 	if (ret)
257 		goto err_rpm_put;
258 
259 	ret = panthor_pwr_init(ptdev);
260 	if (ret)
261 		goto err_rpm_put;
262 
263 	ret = panthor_gpu_init(ptdev);
264 	if (ret)
265 		goto err_unplug_pwr;
266 
267 	ret = panthor_gpu_coherency_init(ptdev);
268 	if (ret)
269 		goto err_unplug_gpu;
270 
271 	ret = panthor_gem_shrinker_init(ptdev);
272 	if (ret)
273 		goto err_unplug_gpu;
274 
275 	ret = panthor_mmu_init(ptdev);
276 	if (ret)
277 		goto err_unplug_shrinker;
278 
279 	ret = panthor_fw_init(ptdev);
280 	if (ret)
281 		goto err_unplug_mmu;
282 
283 	ret = panthor_sched_init(ptdev);
284 	if (ret)
285 		goto err_unplug_fw;
286 
287 	panthor_gem_init(ptdev);
288 
289 	/* Now that everything is initialized, we can enable the reset work. */
290 	enable_work(&ptdev->reset.work);
291 
292 	/* ~3 frames */
293 	pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
294 	pm_runtime_use_autosuspend(ptdev->base.dev);
295 
296 	ret = drm_dev_register(&ptdev->base, 0);
297 	if (ret)
298 		goto err_disable_autosuspend;
299 
300 	pm_runtime_put_autosuspend(ptdev->base.dev);
301 	return 0;
302 
303 err_disable_autosuspend:
304 	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
305 	panthor_sched_unplug(ptdev);
306 
307 err_unplug_fw:
308 	panthor_fw_unplug(ptdev);
309 
310 err_unplug_mmu:
311 	panthor_mmu_unplug(ptdev);
312 
313 err_unplug_shrinker:
314 	panthor_gem_shrinker_unplug(ptdev);
315 
316 err_unplug_gpu:
317 	panthor_gpu_unplug(ptdev);
318 
319 err_unplug_pwr:
320 	panthor_pwr_unplug(ptdev);
321 
322 err_rpm_put:
323 	pm_runtime_put_sync_suspend(ptdev->base.dev);
324 	return ret;
325 }
326 
327 #define PANTHOR_EXCEPTION(id) \
328 	[DRM_PANTHOR_EXCEPTION_ ## id] = { \
329 		.name = #id, \
330 	}
331 
332 struct panthor_exception_info {
333 	const char *name;
334 };
335 
336 static const struct panthor_exception_info panthor_exception_infos[] = {
337 	PANTHOR_EXCEPTION(OK),
338 	PANTHOR_EXCEPTION(TERMINATED),
339 	PANTHOR_EXCEPTION(KABOOM),
340 	PANTHOR_EXCEPTION(EUREKA),
341 	PANTHOR_EXCEPTION(ACTIVE),
342 	PANTHOR_EXCEPTION(CS_RES_TERM),
343 	PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
344 	PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
345 	PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
346 	PANTHOR_EXCEPTION(CS_BUS_FAULT),
347 	PANTHOR_EXCEPTION(CS_INSTR_INVALID),
348 	PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
349 	PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
350 	PANTHOR_EXCEPTION(INSTR_INVALID_PC),
351 	PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
352 	PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
353 	PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
354 	PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
355 	PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
356 	PANTHOR_EXCEPTION(IMPRECISE_FAULT),
357 	PANTHOR_EXCEPTION(OOM),
358 	PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
359 	PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
360 	PANTHOR_EXCEPTION(GPU_BUS_FAULT),
361 	PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
362 	PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
363 	PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
364 	PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
365 	PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
366 	PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
367 	PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
368 	PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
369 	PANTHOR_EXCEPTION(PERM_FAULT_0),
370 	PANTHOR_EXCEPTION(PERM_FAULT_1),
371 	PANTHOR_EXCEPTION(PERM_FAULT_2),
372 	PANTHOR_EXCEPTION(PERM_FAULT_3),
373 	PANTHOR_EXCEPTION(ACCESS_FLAG_1),
374 	PANTHOR_EXCEPTION(ACCESS_FLAG_2),
375 	PANTHOR_EXCEPTION(ACCESS_FLAG_3),
376 	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
377 	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
378 	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
379 	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
380 	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
381 	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
382 	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
383 	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
384 	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
385 };
386 
387 const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
388 {
389 	if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
390 	    !panthor_exception_infos[exception_code].name)
391 		return "Unknown exception type";
392 
393 	return panthor_exception_infos[exception_code].name;
394 }
395 
396 static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
397 {
398 	struct vm_area_struct *vma = vmf->vma;
399 	struct panthor_device *ptdev = vma->vm_private_data;
400 	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
401 	unsigned long pfn;
402 	pgprot_t pgprot;
403 	vm_fault_t ret;
404 	bool active;
405 	int cookie;
406 
407 	if (!drm_dev_enter(&ptdev->base, &cookie))
408 		return VM_FAULT_SIGBUS;
409 
410 	mutex_lock(&ptdev->pm.mmio_lock);
411 	active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;
412 
413 	switch (offset) {
414 	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
415 		if (active)
416 			pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
417 		else
418 			pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
419 		break;
420 
421 	default:
422 		ret = VM_FAULT_SIGBUS;
423 		goto out_unlock;
424 	}
425 
426 	pgprot = vma->vm_page_prot;
427 	if (active)
428 		pgprot = pgprot_noncached(pgprot);
429 
430 	ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);
431 
432 out_unlock:
433 	mutex_unlock(&ptdev->pm.mmio_lock);
434 	drm_dev_exit(cookie);
435 	return ret;
436 }
437 
438 static const struct vm_operations_struct panthor_mmio_vm_ops = {
439 	.fault = panthor_mmio_vm_fault,
440 };
441 
442 int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
443 {
444 	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
445 
446 	if ((vma->vm_flags & VM_SHARED) == 0)
447 		return -EINVAL;
448 
449 	switch (offset) {
450 	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
451 		if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
452 		    (vma->vm_flags & (VM_WRITE | VM_EXEC)))
453 			return -EINVAL;
454 		vm_flags_clear(vma, VM_MAYWRITE);
455 
456 		break;
457 
458 	default:
459 		return -EINVAL;
460 	}
461 
462 	/* Defer actual mapping to the fault handler. */
463 	vma->vm_private_data = ptdev;
464 	vma->vm_ops = &panthor_mmio_vm_ops;
465 	vm_flags_set(vma,
466 		     VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
467 		     VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
468 	return 0;
469 }
470 
471 static int panthor_device_resume_hw_components(struct panthor_device *ptdev)
472 {
473 	int ret;
474 
475 	panthor_pwr_resume(ptdev);
476 	panthor_gpu_resume(ptdev);
477 	panthor_mmu_resume(ptdev);
478 
479 	ret = panthor_fw_resume(ptdev);
480 	if (!ret)
481 		return 0;
482 
483 	panthor_mmu_suspend(ptdev);
484 	panthor_gpu_suspend(ptdev);
485 	panthor_pwr_suspend(ptdev);
486 	return ret;
487 }
488 
489 int panthor_device_resume(struct device *dev)
490 {
491 	struct panthor_device *ptdev = dev_get_drvdata(dev);
492 	int ret, cookie;
493 
494 	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
495 		return -EINVAL;
496 
497 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);
498 
499 	ret = clk_prepare_enable(ptdev->clks.core);
500 	if (ret)
501 		goto err_set_suspended;
502 
503 	ret = clk_prepare_enable(ptdev->clks.stacks);
504 	if (ret)
505 		goto err_disable_core_clk;
506 
507 	ret = clk_prepare_enable(ptdev->clks.coregroup);
508 	if (ret)
509 		goto err_disable_stacks_clk;
510 
511 	panthor_devfreq_resume(ptdev);
512 
513 	if (panthor_device_is_initialized(ptdev) &&
514 	    drm_dev_enter(&ptdev->base, &cookie)) {
515 		/* If there was a reset pending at the time we suspended the
516 		 * device, we force a slow reset.
517 		 */
518 		if (atomic_read(&ptdev->reset.pending)) {
519 			ptdev->reset.fast = false;
520 			atomic_set(&ptdev->reset.pending, 0);
521 		}
522 
523 		ret = panthor_device_resume_hw_components(ptdev);
524 		if (ret && ptdev->reset.fast) {
525 			drm_err(&ptdev->base, "Fast reset failed, trying a slow reset");
526 			ptdev->reset.fast = false;
527 			ret = panthor_device_resume_hw_components(ptdev);
528 		}
529 
530 		if (!ret)
531 			panthor_sched_resume(ptdev);
532 
533 		drm_dev_exit(cookie);
534 
535 		if (ret)
536 			goto err_suspend_devfreq;
537 	}
538 
539 	/* Clear all IOMEM mappings pointing to this device after we've
540 	 * resumed. This way the fake mappings pointing to the dummy pages
541 	 * are removed and the real iomem mapping will be restored on next
542 	 * access.
543 	 */
544 	mutex_lock(&ptdev->pm.mmio_lock);
545 	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
546 			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
547 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
548 	mutex_unlock(&ptdev->pm.mmio_lock);
549 	return 0;
550 
551 err_suspend_devfreq:
552 	panthor_devfreq_suspend(ptdev);
553 	clk_disable_unprepare(ptdev->clks.coregroup);
554 
555 err_disable_stacks_clk:
556 	clk_disable_unprepare(ptdev->clks.stacks);
557 
558 err_disable_core_clk:
559 	clk_disable_unprepare(ptdev->clks.core);
560 
561 err_set_suspended:
562 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
563 	atomic_set(&ptdev->pm.recovery_needed, 1);
564 	return ret;
565 }
566 
567 int panthor_device_suspend(struct device *dev)
568 {
569 	struct panthor_device *ptdev = dev_get_drvdata(dev);
570 	int cookie;
571 
572 	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
573 		return -EINVAL;
574 
575 	/* Clear all IOMEM mappings pointing to this device before we
576 	 * shutdown the power-domain and clocks. Failing to do that results
577 	 * in external aborts when the process accesses the iomem region.
578 	 * We change the state and call unmap_mapping_range() with the
579 	 * mmio_lock held to make sure the vm_fault handler won't set up
580 	 * invalid mappings.
581 	 */
582 	mutex_lock(&ptdev->pm.mmio_lock);
583 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);
584 	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
585 			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
586 	mutex_unlock(&ptdev->pm.mmio_lock);
587 
588 	if (panthor_device_is_initialized(ptdev) &&
589 	    drm_dev_enter(&ptdev->base, &cookie)) {
590 		cancel_work_sync(&ptdev->reset.work);
591 
592 		/* We prepare everything as if we were resetting the GPU.
593 		 * The end of the reset will happen in the resume path though.
594 		 */
595 		panthor_sched_suspend(ptdev);
596 		panthor_fw_suspend(ptdev);
597 		panthor_mmu_suspend(ptdev);
598 		panthor_gpu_suspend(ptdev);
599 		panthor_pwr_suspend(ptdev);
600 		drm_dev_exit(cookie);
601 	}
602 
603 	panthor_devfreq_suspend(ptdev);
604 
605 	clk_disable_unprepare(ptdev->clks.coregroup);
606 	clk_disable_unprepare(ptdev->clks.stacks);
607 	clk_disable_unprepare(ptdev->clks.core);
608 	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
609 	return 0;
610 }
611