25 MODULE_PARM_DESC(address_space_size, "Override for size of processes private GPU address space");
30 static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname,  in zap_shader_load_mdt()  argument
33 	struct device *dev = &gpu->pdev->dev;  in zap_shader_load_mdt()
78 		ret = request_firmware_direct(&fw, fwname, gpu->dev->dev);  in zap_shader_load_mdt()
83 		fw = adreno_request_fw(to_adreno_gpu(gpu), fwname);  in zap_shader_load_mdt()
133 	if (signed_fwname || (to_adreno_gpu(gpu)->fwloc == FW_LOCATION_LEGACY)) {  in zap_shader_load_mdt()
169 int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid)  in adreno_zap_shader_load()  argument
171 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_zap_shader_load()
172 	struct platform_device *pdev = gpu->pdev;  in adreno_zap_shader_load()
184 	return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid);  in adreno_zap_shader_load()
188 adreno_create_vm(struct msm_gpu *gpu,  in adreno_create_vm()  argument
191 	return adreno_iommu_create_vm(gpu, pdev, 0);  in adreno_create_vm()
195 adreno_iommu_create_vm(struct msm_gpu *gpu,  in adreno_iommu_create_vm()  argument
204 	mmu = msm_iommu_gpu_new(&pdev->dev, gpu, quirks);  in adreno_iommu_create_vm()
220 	vm = msm_gem_vm_create(gpu->dev, mmu, "gpu", start & GENMASK_ULL(48, 0),  in adreno_iommu_create_vm()
229 u64 adreno_private_vm_size(struct msm_gpu *gpu)  in adreno_private_vm_size()  argument
231 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_private_vm_size()
232 	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(&gpu->pdev->dev);  in adreno_private_vm_size()
257 	struct msm_gpu *gpu = &adreno_gpu->base;  in adreno_check_and_reenable_stall()  local
258 	struct msm_drm_private *priv = gpu->dev->dev_private;  in adreno_check_and_reenable_stall()
268 			!READ_ONCE(gpu->crashstate)) {  in adreno_check_and_reenable_stall()
269 		struct msm_mmu *mmu = to_msm_vm(gpu->vm)->mmu;  in adreno_check_and_reenable_stall()
283 int adreno_fault_handler(struct msm_gpu *gpu, unsigned long iova, int flags,  in adreno_fault_handler()  argument
287 	struct msm_drm_private *priv = gpu->dev->dev_private;  in adreno_fault_handler()
288 	struct msm_mmu *mmu = to_msm_vm(gpu->vm)->mmu;  in adreno_fault_handler()
291 		!READ_ONCE(gpu->crashstate);  in adreno_fault_handler()
313 		pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",  in adreno_fault_handler()
327 …pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%…  in adreno_fault_handler()
337 		timer_delete(&gpu->hangcheck_timer);  in adreno_fault_handler()
345 		msm_gpu_fault_crashstate_capture(gpu, &fault_info);  in adreno_fault_handler()
351 int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,  in adreno_get_param()  argument
354 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_get_param()
355 	struct drm_device *drm = gpu->dev;  in adreno_get_param()
389 			pm_runtime_get_sync(&gpu->pdev->dev);  in adreno_get_param()
390 			ret = adreno_gpu->funcs->get_timestamp(gpu, value);  in adreno_get_param()
391 			pm_runtime_put_autosuspend(&gpu->pdev->dev);  in adreno_get_param()
397 		*value = gpu->nr_rings * NR_SCHED_PRIORITIES;  in adreno_get_param()
404 			*value = gpu->global_faults + to_msm_vm(vm)->faults;  in adreno_get_param()
406 			*value = gpu->global_faults;  in adreno_get_param()
409 		*value = gpu->suspend_count;  in adreno_get_param()
412 		if (vm == gpu->vm)  in adreno_get_param()
417 		if (vm == gpu->vm)  in adreno_get_param()
437 		*value = adreno_smmu_has_prr(gpu);  in adreno_get_param()
440 		return UERR(EINVAL, drm, "%s: invalid param: %u", gpu->name, param);  in adreno_get_param()
444 int adreno_set_param(struct msm_gpu *gpu, struct msm_context *ctx,  in adreno_set_param()  argument
447 	struct drm_device *drm = gpu->dev;  in adreno_set_param()
472 		mutex_lock(&gpu->lock);  in adreno_set_param()
483 		mutex_unlock(&gpu->lock);  in adreno_set_param()
490 		return msm_context_set_sysprof(ctx, gpu, value);  in adreno_set_param()
493 		if (ctx->vm == gpu->vm)  in adreno_set_param()
507 		return UERR(EINVAL, drm, "%s: invalid param: %u", gpu->name, param);  in adreno_set_param()
620 struct drm_gem_object *adreno_fw_create_bo(struct msm_gpu *gpu,  in adreno_fw_create_bo()  argument
626 	ptr = msm_gem_kernel_new(gpu->dev, fw->size - 4,  in adreno_fw_create_bo()
627 		MSM_BO_WC | MSM_BO_GPU_READONLY, gpu->vm, &bo, iova);  in adreno_fw_create_bo()
639 int adreno_hw_init(struct msm_gpu *gpu)  in adreno_hw_init()  argument
641 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_hw_init()
644 	VERB("%s", gpu->name);  in adreno_hw_init()
651 			DRM_DEV_ERROR(gpu->dev->dev, "unable to set SMMU aperture: %d\n", ret);  in adreno_hw_init()
654 	for (int i = 0; i < gpu->nr_rings; i++) {  in adreno_hw_init()
655 		struct msm_ringbuffer *ring = gpu->rb[i];  in adreno_hw_init()
665 		/* Detect and clean up an impossible fence, ie. if GPU managed  in adreno_hw_init()
681 	struct msm_gpu *gpu = &adreno_gpu->base;  in get_rptr()  local
683 	return gpu->funcs->get_rptr(gpu, ring);  in get_rptr()
686 struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu)  in adreno_active_ring()  argument
688 	return gpu->rb[0];  in adreno_active_ring()
691 void adreno_recover(struct msm_gpu *gpu)  in adreno_recover()  argument
693 	struct drm_device *dev = gpu->dev;  in adreno_recover()
699 	gpu->funcs->pm_suspend(gpu);  in adreno_recover()
700 	gpu->funcs->pm_resume(gpu);  in adreno_recover()
702 	ret = msm_gpu_hw_init(gpu);  in adreno_recover()
704 		DRM_DEV_ERROR(dev->dev, "gpu hw init failed: %d\n", ret);  in adreno_recover()
709 void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring, u32 reg)  in adreno_flush()  argument
726 	gpu_write(gpu, reg, wptr);  in adreno_flush()
729 bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in adreno_idle()  argument
731 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_idle()
738 	/* TODO maybe we need to reset GPU here to recover from hang? */  in adreno_idle()
740 		gpu->name, ring->id, get_rptr(adreno_gpu, ring), wptr);  in adreno_idle()
745 int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state)  in adreno_gpu_state_get()  argument
747 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_gpu_state_get()
750 	WARN_ON(!mutex_is_locked(&gpu->lock));  in adreno_gpu_state_get()
756 	for (i = 0; i < gpu->nr_rings; i++) {  in adreno_gpu_state_get()
759 		state->ring[i].fence = gpu->rb[i]->memptrs->fence;  in adreno_gpu_state_get()
760 		state->ring[i].iova = gpu->rb[i]->iova;  in adreno_gpu_state_get()
761 		state->ring[i].seqno = gpu->rb[i]->fctx->last_fence;  in adreno_gpu_state_get()
762 		state->ring[i].rptr = get_rptr(adreno_gpu, gpu->rb[i]);  in adreno_gpu_state_get()
763 		state->ring[i].wptr = get_wptr(gpu->rb[i]);  in adreno_gpu_state_get()
770 			if (gpu->rb[i]->start[j])  in adreno_gpu_state_get()
774 			state->ring[i].data = kvmemdup(gpu->rb[i]->start, size << 2, GFP_KERNEL);  in adreno_gpu_state_get()
800 				state->registers[pos++] = gpu_read(gpu, addr);  in adreno_gpu_state_get()
923 void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,  in adreno_show()  argument
926 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_show()
975 	for (i = 0; i < gpu->nr_rings; i++) {  in adreno_show()
1015 /* Dump common gpu status and scratch registers on any hang, to make
1017  * safe to read when GPU has hung (unlike some other regs, depending
1018  * on how the GPU hung), and they are useful to match up to cmdstream
1021 void adreno_dump_info(struct msm_gpu *gpu)  in adreno_dump_info()  argument
1023 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_dump_info()
1030 	for (i = 0; i < gpu->nr_rings; i++) {  in adreno_dump_info()
1031 		struct msm_ringbuffer *ring = gpu->rb[i];  in adreno_dump_info()
1043 void adreno_dump(struct msm_gpu *gpu)  in adreno_dump()  argument
1045 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_dump()
1052 	printk("IO:region %s 00000000 00020000\n", gpu->name);  in adreno_dump()
1059 			uint32_t val = gpu_read(gpu, addr);  in adreno_dump()
1067 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(ring->gpu);  in ring_freewords()
1078 		DRM_DEV_ERROR(ring->gpu->dev->dev,  in adreno_wait_ring()
1084 		struct msm_gpu *gpu)  in adreno_get_pwrlevels()  argument
1086 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in adreno_get_pwrlevels()
1091 	gpu->fast_rate = 0;  in adreno_get_pwrlevels()
1117 	gpu->fast_rate = freq;  in adreno_get_pwrlevels()
1120 	DBG("fast_rate=%u, slow_rate=27000000", gpu->fast_rate);  in adreno_get_pwrlevels()
1178 	struct msm_gpu *gpu = &adreno_gpu->base;  in adreno_gpu_init()  local
1187 	gpu->allow_relocs = config->info->family < ADRENO_6XX_GEN1;  in adreno_gpu_init()
1188 	gpu->pdev = pdev;  in adreno_gpu_init()
1221 	ret = adreno_get_pwrlevels(dev, gpu);  in adreno_gpu_init()
1235 	struct msm_gpu *gpu = &adreno_gpu->base;  in adreno_gpu_cleanup()  local
1236 	struct msm_drm_private *priv = gpu->dev ? gpu->dev->dev_private : NULL;  in adreno_gpu_cleanup()