25 static int enable_pwrrail(struct msm_gpu *gpu)  in enable_pwrrail()  argument
27 	struct drm_device *dev = gpu->dev;  in enable_pwrrail()
30 	if (gpu->gpu_reg) {  in enable_pwrrail()
31 		ret = regulator_enable(gpu->gpu_reg);  in enable_pwrrail()
38 	if (gpu->gpu_cx) {  in enable_pwrrail()
39 		ret = regulator_enable(gpu->gpu_cx);  in enable_pwrrail()
49 static int disable_pwrrail(struct msm_gpu *gpu)  in disable_pwrrail()  argument
51 	if (gpu->gpu_cx)  in disable_pwrrail()
52 		regulator_disable(gpu->gpu_cx);  in disable_pwrrail()
53 	if (gpu->gpu_reg)  in disable_pwrrail()
54 		regulator_disable(gpu->gpu_reg);  in disable_pwrrail()
58 static int enable_clk(struct msm_gpu *gpu)  in enable_clk()  argument
60 	if (gpu->core_clk && gpu->fast_rate)  in enable_clk()
61 		dev_pm_opp_set_rate(&gpu->pdev->dev, gpu->fast_rate);  in enable_clk()
64 	if (gpu->rbbmtimer_clk)  in enable_clk()
65 		clk_set_rate(gpu->rbbmtimer_clk, 19200000);  in enable_clk()
67 	return clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);  in enable_clk()
70 static int disable_clk(struct msm_gpu *gpu)  in disable_clk()  argument
72 	clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks);  in disable_clk()
79 	if (gpu->core_clk)  in disable_clk()
80 		dev_pm_opp_set_rate(&gpu->pdev->dev, 27000000);  in disable_clk()
82 	if (gpu->rbbmtimer_clk)  in disable_clk()
83 		clk_set_rate(gpu->rbbmtimer_clk, 0);  in disable_clk()
88 static int enable_axi(struct msm_gpu *gpu)  in enable_axi()  argument
90 	return clk_prepare_enable(gpu->ebi1_clk);  in enable_axi()
93 static int disable_axi(struct msm_gpu *gpu)  in disable_axi()  argument
95 	clk_disable_unprepare(gpu->ebi1_clk);  in disable_axi()
99 int msm_gpu_pm_resume(struct msm_gpu *gpu)  in msm_gpu_pm_resume()  argument
103 	DBG("%s", gpu->name);  in msm_gpu_pm_resume()
106 	ret = enable_pwrrail(gpu);  in msm_gpu_pm_resume()
110 	ret = enable_clk(gpu);  in msm_gpu_pm_resume()
114 	ret = enable_axi(gpu);  in msm_gpu_pm_resume()
118 	msm_devfreq_resume(gpu);  in msm_gpu_pm_resume()
120 	gpu->needs_hw_init = true;  in msm_gpu_pm_resume()
125 int msm_gpu_pm_suspend(struct msm_gpu *gpu)  in msm_gpu_pm_suspend()  argument
129 	DBG("%s", gpu->name);  in msm_gpu_pm_suspend()
132 	msm_devfreq_suspend(gpu);  in msm_gpu_pm_suspend()
134 	ret = disable_axi(gpu);  in msm_gpu_pm_suspend()
138 	ret = disable_clk(gpu);  in msm_gpu_pm_suspend()
142 	ret = disable_pwrrail(gpu);  in msm_gpu_pm_suspend()
146 	gpu->suspend_count++;  in msm_gpu_pm_suspend()
151 void msm_gpu_show_fdinfo(struct msm_gpu *gpu, struct msm_context *ctx,  in msm_gpu_show_fdinfo()  argument
154 	drm_printf(p, "drm-engine-gpu:\t%llu ns\n", ctx->elapsed_ns);  in msm_gpu_show_fdinfo()
155 	drm_printf(p, "drm-cycles-gpu:\t%llu\n", ctx->cycles);  in msm_gpu_show_fdinfo()
156 	drm_printf(p, "drm-maxfreq-gpu:\t%u Hz\n", gpu->fast_rate);  in msm_gpu_show_fdinfo()
159 int msm_gpu_hw_init(struct msm_gpu *gpu)  in msm_gpu_hw_init()  argument
163 	WARN_ON(!mutex_is_locked(&gpu->lock));  in msm_gpu_hw_init()
165 	if (!gpu->needs_hw_init)  in msm_gpu_hw_init()
168 	disable_irq(gpu->irq);  in msm_gpu_hw_init()
169 	ret = gpu->funcs->hw_init(gpu);  in msm_gpu_hw_init()
171 		gpu->needs_hw_init = false;  in msm_gpu_hw_init()
172 	enable_irq(gpu->irq);  in msm_gpu_hw_init()
181 	struct msm_gpu *gpu = data;  in msm_gpu_devcoredump_read()  local
186 	state = msm_gpu_crashstate_get(gpu);  in msm_gpu_devcoredump_read()
207 	gpu->funcs->show(gpu, state, &p);  in msm_gpu_devcoredump_read()
209 	msm_gpu_crashstate_put(gpu);  in msm_gpu_devcoredump_read()
216 	struct msm_gpu *gpu = data;  in msm_gpu_devcoredump_free()  local
218 	msm_gpu_crashstate_put(gpu);  in msm_gpu_devcoredump_free()
360 static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,  in msm_gpu_crashstate_capture()  argument
367 	if (!gpu->funcs->gpu_state_get)  in msm_gpu_crashstate_capture()
371 	if (gpu->crashstate)  in msm_gpu_crashstate_capture()
374 	state = gpu->funcs->gpu_state_get(gpu);  in msm_gpu_crashstate_capture()
399 	gpu->crashstate = state;  in msm_gpu_crashstate_capture()
401 	dev_coredumpm(&gpu->pdev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL,  in msm_gpu_crashstate_capture()
405 static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,  in msm_gpu_crashstate_capture()  argument
413  * Hangcheck detection for locked gpu:
434 static void retire_submits(struct msm_gpu *gpu);
441 	WARN_ON(!mutex_is_locked(&submit->gpu->lock));  in get_comm_cmdline()
462 	struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);  in recover_worker()  local
463 	struct drm_device *dev = gpu->dev;  in recover_worker()
466 	struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);  in recover_worker()
471 	mutex_lock(&gpu->lock);  in recover_worker()
473 	DRM_DEV_ERROR(dev->dev, "%s: hangcheck recover!\n", gpu->name);  in recover_worker()
479 	 * or waiting to acquire the gpu lock, then nothing more to do.  in recover_worker()
489 		gpu->global_faults++;  in recover_worker()
509 			      gpu->name, comm, cmd);  in recover_worker()
514 		DRM_DEV_ERROR(dev->dev, "%s: offending task: unknown\n", gpu->name);  in recover_worker()
520 	pm_runtime_get_sync(&gpu->pdev->dev);  in recover_worker()
521 	msm_gpu_crashstate_capture(gpu, submit, NULL, comm, cmd);  in recover_worker()
531 	for (i = 0; i < gpu->nr_rings; i++) {  in recover_worker()
532 		struct msm_ringbuffer *ring = gpu->rb[i];  in recover_worker()
546 	if (msm_gpu_active(gpu)) {  in recover_worker()
548 		retire_submits(gpu);  in recover_worker()
550 		gpu->funcs->recover(gpu);  in recover_worker()
556 		for (i = 0; i < gpu->nr_rings; i++) {  in recover_worker()
557 			struct msm_ringbuffer *ring = gpu->rb[i];  in recover_worker()
568 				gpu->funcs->submit(gpu, submit);  in recover_worker()
574 	pm_runtime_put(&gpu->pdev->dev);  in recover_worker()
577 	mutex_unlock(&gpu->lock);  in recover_worker()
579 	msm_gpu_retire(gpu);  in recover_worker()
582 void msm_gpu_fault_crashstate_capture(struct msm_gpu *gpu, struct msm_gpu_fault_info *fault_info)  in msm_gpu_fault_crashstate_capture()  argument
585 	struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);  in msm_gpu_fault_crashstate_capture()
588 	mutex_lock(&gpu->lock);  in msm_gpu_fault_crashstate_capture()
598 		 * When we get GPU iova faults, we can get 1000s of them,  in msm_gpu_fault_crashstate_capture()
605 	pm_runtime_get_sync(&gpu->pdev->dev);  in msm_gpu_fault_crashstate_capture()
606 	msm_gpu_crashstate_capture(gpu, submit, fault_info, comm, cmd);  in msm_gpu_fault_crashstate_capture()
607 	pm_runtime_put_sync(&gpu->pdev->dev);  in msm_gpu_fault_crashstate_capture()
613 	mutex_unlock(&gpu->lock);  in msm_gpu_fault_crashstate_capture()
616 static void hangcheck_timer_reset(struct msm_gpu *gpu)  in hangcheck_timer_reset()  argument
618 	struct msm_drm_private *priv = gpu->dev->dev_private;  in hangcheck_timer_reset()
619 	mod_timer(&gpu->hangcheck_timer,  in hangcheck_timer_reset()
623 static bool made_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in made_progress()  argument
628 	if (!gpu->funcs->progress)  in made_progress()
631 	if (!gpu->funcs->progress(gpu, ring))  in made_progress()
640 	struct msm_gpu *gpu = timer_container_of(gpu, t, hangcheck_timer);  in hangcheck_handler()  local
641 	struct drm_device *dev = gpu->dev;  in hangcheck_handler()
642 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in hangcheck_handler()
650 			!made_progress(gpu, ring)) {  in hangcheck_handler()
654 		DRM_DEV_ERROR(dev->dev, "%s: hangcheck detected gpu lockup rb %d!\n",  in hangcheck_handler()
655 				gpu->name, ring->id);  in hangcheck_handler()
657 				gpu->name, fence);  in hangcheck_handler()
659 				gpu->name, ring->fctx->last_fence);  in hangcheck_handler()
661 		kthread_queue_work(gpu->worker, &gpu->recover_work);  in hangcheck_handler()
666 		hangcheck_timer_reset(gpu);  in hangcheck_handler()
669 	msm_gpu_retire(gpu);  in hangcheck_handler()
677 static int update_hw_cntrs(struct msm_gpu *gpu, uint32_t ncntrs, uint32_t *cntrs)  in update_hw_cntrs()  argument
679 	uint32_t current_cntrs[ARRAY_SIZE(gpu->last_cntrs)];  in update_hw_cntrs()
680 	int i, n = min(ncntrs, gpu->num_perfcntrs);  in update_hw_cntrs()
683 	for (i = 0; i < gpu->num_perfcntrs; i++)  in update_hw_cntrs()
684 		current_cntrs[i] = gpu_read(gpu, gpu->perfcntrs[i].sample_reg);  in update_hw_cntrs()
688 		cntrs[i] = current_cntrs[i] - gpu->last_cntrs[i];  in update_hw_cntrs()
691 	for (i = 0; i < gpu->num_perfcntrs; i++)  in update_hw_cntrs()
692 		gpu->last_cntrs[i] = current_cntrs[i];  in update_hw_cntrs()
697 static void update_sw_cntrs(struct msm_gpu *gpu)  in update_sw_cntrs()  argument
703 	spin_lock_irqsave(&gpu->perf_lock, flags);  in update_sw_cntrs()
704 	if (!gpu->perfcntr_active)  in update_sw_cntrs()
708 	elapsed = ktime_to_us(ktime_sub(time, gpu->last_sample.time));  in update_sw_cntrs()
710 	gpu->totaltime += elapsed;  in update_sw_cntrs()
711 	if (gpu->last_sample.active)  in update_sw_cntrs()
712 		gpu->activetime += elapsed;  in update_sw_cntrs()
714 	gpu->last_sample.active = msm_gpu_active(gpu);  in update_sw_cntrs()
715 	gpu->last_sample.time = time;  in update_sw_cntrs()
718 	spin_unlock_irqrestore(&gpu->perf_lock, flags);  in update_sw_cntrs()
721 void msm_gpu_perfcntr_start(struct msm_gpu *gpu)  in msm_gpu_perfcntr_start()  argument
725 	pm_runtime_get_sync(&gpu->pdev->dev);  in msm_gpu_perfcntr_start()
727 	spin_lock_irqsave(&gpu->perf_lock, flags);  in msm_gpu_perfcntr_start()
729 	gpu->last_sample.active = msm_gpu_active(gpu);  in msm_gpu_perfcntr_start()
730 	gpu->last_sample.time = ktime_get();  in msm_gpu_perfcntr_start()
731 	gpu->activetime = gpu->totaltime = 0;  in msm_gpu_perfcntr_start()
732 	gpu->perfcntr_active = true;  in msm_gpu_perfcntr_start()
733 	update_hw_cntrs(gpu, 0, NULL);  in msm_gpu_perfcntr_start()
734 	spin_unlock_irqrestore(&gpu->perf_lock, flags);  in msm_gpu_perfcntr_start()
737 void msm_gpu_perfcntr_stop(struct msm_gpu *gpu)  in msm_gpu_perfcntr_stop()  argument
739 	gpu->perfcntr_active = false;  in msm_gpu_perfcntr_stop()
740 	pm_runtime_put_sync(&gpu->pdev->dev);  in msm_gpu_perfcntr_stop()
744 int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,  in msm_gpu_perfcntr_sample()  argument
750 	spin_lock_irqsave(&gpu->perf_lock, flags);  in msm_gpu_perfcntr_sample()
752 	if (!gpu->perfcntr_active) {  in msm_gpu_perfcntr_sample()
757 	*activetime = gpu->activetime;  in msm_gpu_perfcntr_sample()
758 	*totaltime = gpu->totaltime;  in msm_gpu_perfcntr_sample()
760 	gpu->activetime = gpu->totaltime = 0;  in msm_gpu_perfcntr_sample()
762 	ret = update_hw_cntrs(gpu, ncntrs, cntrs);  in msm_gpu_perfcntr_sample()
765 	spin_unlock_irqrestore(&gpu->perf_lock, flags);  in msm_gpu_perfcntr_sample()
774 static void retire_submit(struct msm_gpu *gpu, struct msm_ringbuffer *ring,  in retire_submit()  argument
803 	pm_runtime_mark_last_busy(&gpu->pdev->dev);  in retire_submit()
810 	mutex_lock(&gpu->active_lock);  in retire_submit()
811 	gpu->active_submits--;  in retire_submit()
812 	WARN_ON(gpu->active_submits < 0);  in retire_submit()
813 	if (!gpu->active_submits) {  in retire_submit()
814 		msm_devfreq_idle(gpu);  in retire_submit()
815 		pm_runtime_put_autosuspend(&gpu->pdev->dev);  in retire_submit()
818 	mutex_unlock(&gpu->active_lock);  in retire_submit()
823 static void retire_submits(struct msm_gpu *gpu)  in retire_submits()  argument
828 	for (i = 0; i < gpu->nr_rings; i++) {  in retire_submits()
829 		struct msm_ringbuffer *ring = gpu->rb[i];  in retire_submits()
846 				retire_submit(gpu, ring, submit);  in retire_submits()
853 	wake_up_all(&gpu->retire_event);  in retire_submits()
858 	struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);  in retire_worker()  local
860 	retire_submits(gpu);  in retire_worker()
864 void msm_gpu_retire(struct msm_gpu *gpu)  in msm_gpu_retire()  argument
868 	for (i = 0; i < gpu->nr_rings; i++)  in msm_gpu_retire()
869 		msm_update_fence(gpu->rb[i]->fctx, gpu->rb[i]->memptrs->fence);  in msm_gpu_retire()
871 	kthread_queue_work(gpu->worker, &gpu->retire_work);  in msm_gpu_retire()
872 	update_sw_cntrs(gpu);  in msm_gpu_retire()
875 /* add bo's to gpu's ring, and kick gpu: */
876 void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)  in msm_gpu_submit()  argument
881 	WARN_ON(!mutex_is_locked(&gpu->lock));  in msm_gpu_submit()
883 	pm_runtime_get_sync(&gpu->pdev->dev);  in msm_gpu_submit()
885 	msm_gpu_hw_init(gpu);  in msm_gpu_submit()
889 	update_sw_cntrs(gpu);  in msm_gpu_submit()
902 	mutex_lock(&gpu->active_lock);  in msm_gpu_submit()
903 	if (!gpu->active_submits) {  in msm_gpu_submit()
904 		pm_runtime_get(&gpu->pdev->dev);  in msm_gpu_submit()
905 		msm_devfreq_active(gpu);  in msm_gpu_submit()
907 	gpu->active_submits++;  in msm_gpu_submit()
908 	mutex_unlock(&gpu->active_lock);  in msm_gpu_submit()
910 	gpu->funcs->submit(gpu, submit);  in msm_gpu_submit()
913 	pm_runtime_put(&gpu->pdev->dev);  in msm_gpu_submit()
914 	hangcheck_timer_reset(gpu);  in msm_gpu_submit()
923 	struct msm_gpu *gpu = data;  in irq_handler()  local
924 	return gpu->funcs->irq(gpu);  in irq_handler()
927 static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)  in get_clocks()  argument
929 	int ret = devm_clk_bulk_get_all(&pdev->dev, &gpu->grp_clks);  in get_clocks()
932 		gpu->nr_clocks = 0;  in get_clocks()
936 	gpu->nr_clocks = ret;  in get_clocks()
938 	gpu->core_clk = msm_clk_bulk_get_clock(gpu->grp_clks,  in get_clocks()
939 		gpu->nr_clocks, "core");  in get_clocks()
941 	gpu->rbbmtimer_clk = msm_clk_bulk_get_clock(gpu->grp_clks,  in get_clocks()
942 		gpu->nr_clocks, "rbbmtimer");  in get_clocks()
949 msm_gpu_create_private_vm(struct msm_gpu *gpu, struct task_struct *task,  in msm_gpu_create_private_vm()  argument
954 	if (!gpu)  in msm_gpu_create_private_vm()
961 	if (gpu->funcs->create_private_vm) {  in msm_gpu_create_private_vm()
962 		vm = gpu->funcs->create_private_vm(gpu, kernel_managed);  in msm_gpu_create_private_vm()
968 		vm = drm_gpuvm_get(gpu->vm);  in msm_gpu_create_private_vm()
974 		struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,  in msm_gpu_init()  argument
982 	if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs)))  in msm_gpu_init()
983 		gpu->num_perfcntrs = ARRAY_SIZE(gpu->last_cntrs);  in msm_gpu_init()
985 	gpu->dev = drm;  in msm_gpu_init()
986 	gpu->funcs = funcs;  in msm_gpu_init()
987 	gpu->name = name;  in msm_gpu_init()
989 	gpu->worker = kthread_run_worker(0, "gpu-worker");  in msm_gpu_init()
990 	if (IS_ERR(gpu->worker)) {  in msm_gpu_init()
991 		ret = PTR_ERR(gpu->worker);  in msm_gpu_init()
992 		gpu->worker = NULL;  in msm_gpu_init()
996 	sched_set_fifo_low(gpu->worker->task);  in msm_gpu_init()
998 	mutex_init(&gpu->active_lock);  in msm_gpu_init()
999 	mutex_init(&gpu->lock);  in msm_gpu_init()
1000 	init_waitqueue_head(&gpu->retire_event);  in msm_gpu_init()
1001 	kthread_init_work(&gpu->retire_work, retire_worker);  in msm_gpu_init()
1002 	kthread_init_work(&gpu->recover_work, recover_worker);  in msm_gpu_init()
1014 	timer_setup(&gpu->hangcheck_timer, hangcheck_handler, 0);  in msm_gpu_init()
1016 	spin_lock_init(&gpu->perf_lock);  in msm_gpu_init()
1020 	gpu->mmio = msm_ioremap(pdev, config->ioname);  in msm_gpu_init()
1021 	if (IS_ERR(gpu->mmio)) {  in msm_gpu_init()
1022 		ret = PTR_ERR(gpu->mmio);  in msm_gpu_init()
1027 	gpu->irq = platform_get_irq(pdev, 0);  in msm_gpu_init()
1028 	if (gpu->irq < 0) {  in msm_gpu_init()
1029 		ret = gpu->irq;  in msm_gpu_init()
1033 	ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler,  in msm_gpu_init()
1034 			IRQF_TRIGGER_HIGH, "gpu-irq", gpu);  in msm_gpu_init()
1036 		DRM_DEV_ERROR(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);  in msm_gpu_init()
1040 	ret = get_clocks(pdev, gpu);  in msm_gpu_init()
1044 	gpu->ebi1_clk = msm_clk_get(pdev, "bus");  in msm_gpu_init()
1045 	DBG("ebi1_clk: %p", gpu->ebi1_clk);  in msm_gpu_init()
1046 	if (IS_ERR(gpu->ebi1_clk))  in msm_gpu_init()
1047 		gpu->ebi1_clk = NULL;  in msm_gpu_init()
1050 	gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd");  in msm_gpu_init()
1051 	DBG("gpu_reg: %p", gpu->gpu_reg);  in msm_gpu_init()
1052 	if (IS_ERR(gpu->gpu_reg))  in msm_gpu_init()
1053 		gpu->gpu_reg = NULL;  in msm_gpu_init()
1055 	gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx");  in msm_gpu_init()
1056 	DBG("gpu_cx: %p", gpu->gpu_cx);  in msm_gpu_init()
1057 	if (IS_ERR(gpu->gpu_cx))  in msm_gpu_init()
1058 		gpu->gpu_cx = NULL;  in msm_gpu_init()
1060 	platform_set_drvdata(pdev, &gpu->adreno_smmu);  in msm_gpu_init()
1062 	msm_devfreq_init(gpu);  in msm_gpu_init()
1064 	gpu->vm = gpu->funcs->create_vm(gpu, pdev);  in msm_gpu_init()
1065 	if (IS_ERR(gpu->vm)) {  in msm_gpu_init()
1066 		ret = PTR_ERR(gpu->vm);  in msm_gpu_init()
1072 		check_apriv(gpu, MSM_BO_WC), gpu->vm, &gpu->memptrs_bo,  in msm_gpu_init()
1081 	msm_gem_object_set_name(gpu->memptrs_bo, "memptrs");  in msm_gpu_init()
1083 	if (nr_rings > ARRAY_SIZE(gpu->rb)) {  in msm_gpu_init()
1085 			ARRAY_SIZE(gpu->rb));  in msm_gpu_init()
1086 		nr_rings = ARRAY_SIZE(gpu->rb);  in msm_gpu_init()
1091 		gpu->rb[i] = msm_ringbuffer_new(gpu, i, memptrs, memptrs_iova);  in msm_gpu_init()
1093 		if (IS_ERR(gpu->rb[i])) {  in msm_gpu_init()
1094 			ret = PTR_ERR(gpu->rb[i]);  in msm_gpu_init()
1104 	gpu->nr_rings = nr_rings;  in msm_gpu_init()
1106 	refcount_set(&gpu->sysprof_active, 1);  in msm_gpu_init()
1111 	for (i = 0; i < ARRAY_SIZE(gpu->rb); i++)  {  in msm_gpu_init()
1112 		msm_ringbuffer_destroy(gpu->rb[i]);  in msm_gpu_init()
1113 		gpu->rb[i] = NULL;  in msm_gpu_init()
1116 	msm_gem_kernel_put(gpu->memptrs_bo, gpu->vm);  in msm_gpu_init()
1122 void msm_gpu_cleanup(struct msm_gpu *gpu)  in msm_gpu_cleanup()  argument
1126 	DBG("%s", gpu->name);  in msm_gpu_cleanup()
1128 	for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {  in msm_gpu_cleanup()
1129 		msm_ringbuffer_destroy(gpu->rb[i]);  in msm_gpu_cleanup()
1130 		gpu->rb[i] = NULL;  in msm_gpu_cleanup()
1133 	msm_gem_kernel_put(gpu->memptrs_bo, gpu->vm);  in msm_gpu_cleanup()
1135 	if (!IS_ERR_OR_NULL(gpu->vm)) {  in msm_gpu_cleanup()
1136 		struct msm_mmu *mmu = to_msm_vm(gpu->vm)->mmu;  in msm_gpu_cleanup()
1138 		drm_gpuvm_put(gpu->vm);  in msm_gpu_cleanup()
1141 	if (gpu->worker) {  in msm_gpu_cleanup()
1142 		kthread_destroy_worker(gpu->worker);  in msm_gpu_cleanup()
1145 	msm_devfreq_cleanup(gpu);  in msm_gpu_cleanup()
1147 	platform_set_drvdata(gpu->pdev, NULL);  in msm_gpu_cleanup()