1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2019 Collabora Ltd */ 3 4 #include <drm/drm_file.h> 5 #include <drm/drm_gem_shmem_helper.h> 6 #include <drm/panfrost_drm.h> 7 #include <linux/completion.h> 8 #include <linux/iopoll.h> 9 #include <linux/pm_runtime.h> 10 #include <linux/slab.h> 11 #include <linux/uaccess.h> 12 13 #include "panfrost_device.h" 14 #include "panfrost_features.h" 15 #include "panfrost_gem.h" 16 #include "panfrost_issues.h" 17 #include "panfrost_job.h" 18 #include "panfrost_mmu.h" 19 #include "panfrost_perfcnt.h" 20 #include "panfrost_regs.h" 21 22 #define COUNTERS_PER_BLOCK 64 23 #define BYTES_PER_COUNTER 4 24 #define BLOCKS_PER_COREGROUP 8 25 #define V4_SHADERS_PER_COREGROUP 4 26 27 struct panfrost_perfcnt { 28 struct panfrost_gem_object *bo; 29 size_t bosize; 30 void *buf; 31 struct panfrost_file_priv *user; 32 struct mutex lock; 33 struct completion dump_comp; 34 }; 35 36 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev) 37 { 38 complete(&pfdev->perfcnt->dump_comp); 39 } 40 41 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev) 42 { 43 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES); 44 } 45 46 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev) 47 { 48 u64 gpuva; 49 int ret; 50 51 reinit_completion(&pfdev->perfcnt->dump_comp); 52 gpuva = pfdev->perfcnt->bo->node.start << PAGE_SHIFT; 53 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva); 54 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32); 55 gpu_write(pfdev, GPU_INT_CLEAR, 56 GPU_IRQ_CLEAN_CACHES_COMPLETED | 57 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 58 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE); 59 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp, 60 msecs_to_jiffies(1000)); 61 if (!ret) 62 ret = -ETIMEDOUT; 63 else if (ret > 0) 64 ret = 0; 65 66 return ret; 67 } 68 69 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, 70 struct panfrost_file_priv *user, 71 unsigned int counterset) 72 { 73 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 74 struct drm_gem_shmem_object *bo; 75 u32 cfg; 76 int ret; 77 78 if (user == perfcnt->user) 79 return 0; 80 else if (perfcnt->user) 81 return -EBUSY; 82 83 ret = pm_runtime_get_sync(pfdev->dev); 84 if (ret < 0) 85 return ret; 86 87 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize); 88 if (IS_ERR(bo)) 89 return PTR_ERR(bo); 90 91 perfcnt->bo = to_panfrost_bo(&bo->base); 92 93 /* Map the perfcnt buf in the address space attached to file_priv. */ 94 ret = panfrost_mmu_map(perfcnt->bo); 95 if (ret) 96 goto err_put_bo; 97 98 perfcnt->buf = drm_gem_shmem_vmap(&bo->base); 99 if (IS_ERR(perfcnt->buf)) { 100 ret = PTR_ERR(perfcnt->buf); 101 goto err_put_bo; 102 } 103 104 /* 105 * Invalidate the cache and clear the counters to start from a fresh 106 * state. 107 */ 108 reinit_completion(&pfdev->perfcnt->dump_comp); 109 gpu_write(pfdev, GPU_INT_CLEAR, 110 GPU_IRQ_CLEAN_CACHES_COMPLETED | 111 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 112 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR); 113 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES); 114 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp, 115 msecs_to_jiffies(1000)); 116 if (!ret) { 117 ret = -ETIMEDOUT; 118 goto err_vunmap; 119 } 120 121 perfcnt->user = user; 122 123 /* 124 * Always use address space 0 for now. 125 * FIXME: this needs to be updated when we start using different 126 * address space. 127 */ 128 cfg = GPU_PERFCNT_CFG_AS(0) | 129 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL); 130 131 /* 132 * Bifrost GPUs have 2 set of counters, but we're only interested by 133 * the first one for now. 134 */ 135 if (panfrost_model_is_bifrost(pfdev)) 136 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset); 137 138 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff); 139 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff); 140 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff); 141 142 /* 143 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW 144 * counters. 145 */ 146 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 147 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 148 else 149 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 150 151 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg); 152 153 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 154 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 155 156 return 0; 157 158 err_vunmap: 159 drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf); 160 err_put_bo: 161 drm_gem_object_put_unlocked(&bo->base); 162 return ret; 163 } 164 165 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, 166 struct panfrost_file_priv *user) 167 { 168 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 169 170 if (user != perfcnt->user) 171 return -EINVAL; 172 173 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0); 174 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0); 175 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0); 176 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 177 gpu_write(pfdev, GPU_PERFCNT_CFG, 178 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 179 180 perfcnt->user = NULL; 181 drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf); 182 perfcnt->buf = NULL; 183 drm_gem_object_put_unlocked(&perfcnt->bo->base.base); 184 perfcnt->bo = NULL; 185 pm_runtime_mark_last_busy(pfdev->dev); 186 pm_runtime_put_autosuspend(pfdev->dev); 187 188 return 0; 189 } 190 191 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, 192 struct drm_file *file_priv) 193 { 194 struct panfrost_file_priv *pfile = file_priv->driver_priv; 195 struct panfrost_device *pfdev = dev->dev_private; 196 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 197 struct drm_panfrost_perfcnt_enable *req = data; 198 int ret; 199 200 ret = panfrost_unstable_ioctl_check(); 201 if (ret) 202 return ret; 203 204 /* Only Bifrost GPUs have 2 set of counters. */ 205 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0)) 206 return -EINVAL; 207 208 mutex_lock(&perfcnt->lock); 209 if (req->enable) 210 ret = panfrost_perfcnt_enable_locked(pfdev, pfile, 211 req->counterset); 212 else 213 ret = panfrost_perfcnt_disable_locked(pfdev, pfile); 214 mutex_unlock(&perfcnt->lock); 215 216 return ret; 217 } 218 219 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, 220 struct drm_file *file_priv) 221 { 222 struct panfrost_device *pfdev = dev->dev_private; 223 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 224 struct drm_panfrost_perfcnt_dump *req = data; 225 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr; 226 int ret; 227 228 ret = panfrost_unstable_ioctl_check(); 229 if (ret) 230 return ret; 231 232 mutex_lock(&perfcnt->lock); 233 if (perfcnt->user != file_priv->driver_priv) { 234 ret = -EINVAL; 235 goto out; 236 } 237 238 ret = panfrost_perfcnt_dump_locked(pfdev); 239 if (ret) 240 goto out; 241 242 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize)) 243 ret = -EFAULT; 244 245 out: 246 mutex_unlock(&perfcnt->lock); 247 248 return ret; 249 } 250 251 void panfrost_perfcnt_close(struct panfrost_file_priv *pfile) 252 { 253 struct panfrost_device *pfdev = pfile->pfdev; 254 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 255 256 pm_runtime_get_sync(pfdev->dev); 257 mutex_lock(&perfcnt->lock); 258 if (perfcnt->user == pfile) 259 panfrost_perfcnt_disable_locked(pfdev, pfile); 260 mutex_unlock(&perfcnt->lock); 261 pm_runtime_mark_last_busy(pfdev->dev); 262 pm_runtime_put_autosuspend(pfdev->dev); 263 } 264 265 int panfrost_perfcnt_init(struct panfrost_device *pfdev) 266 { 267 struct panfrost_perfcnt *perfcnt; 268 size_t size; 269 270 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) { 271 unsigned int ncoregroups; 272 273 ncoregroups = hweight64(pfdev->features.l2_present); 274 size = ncoregroups * BLOCKS_PER_COREGROUP * 275 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 276 } else { 277 unsigned int nl2c, ncores; 278 279 /* 280 * TODO: define a macro to extract the number of l2 caches from 281 * mem_features. 282 */ 283 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1; 284 285 /* 286 * shader_present might be sparse, but the counters layout 287 * forces to dump unused regions too, hence the fls64() call 288 * instead of hweight64(). 289 */ 290 ncores = fls64(pfdev->features.shader_present); 291 292 /* 293 * There's always one JM and one Tiler block, hence the '+ 2' 294 * here. 295 */ 296 size = (nl2c + ncores + 2) * 297 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 298 } 299 300 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL); 301 if (!perfcnt) 302 return -ENOMEM; 303 304 perfcnt->bosize = size; 305 306 /* Start with everything disabled. */ 307 gpu_write(pfdev, GPU_PERFCNT_CFG, 308 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 309 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 310 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 311 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 312 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 313 314 init_completion(&perfcnt->dump_comp); 315 mutex_init(&perfcnt->lock); 316 pfdev->perfcnt = perfcnt; 317 318 return 0; 319 } 320 321 void panfrost_perfcnt_fini(struct panfrost_device *pfdev) 322 { 323 /* Disable everything before leaving. */ 324 gpu_write(pfdev, GPU_PERFCNT_CFG, 325 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 326 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 327 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 328 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 329 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 330 } 331